From 425f5bef8e6bc1356d89f4369f243368401a52f7 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Thu, 28 Jul 2016 11:55:29 -0400 Subject: [PATCH 1/9] adding support for each searchindex class to specific which task should consume the celery message in order to optimize certain celery queues --- celery_haystack/indexes.py | 2 +- celery_haystack/signals.py | 2 +- celery_haystack/utils.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery_haystack/indexes.py b/celery_haystack/indexes.py index 07dbf90..c4499a1 100644 --- a/celery_haystack/indexes.py +++ b/celery_haystack/indexes.py @@ -2,4 +2,4 @@ class CelerySearchIndex(indexes.SearchIndex): - pass + task_path = None diff --git a/celery_haystack/signals.py b/celery_haystack/signals.py index f952f99..682221f 100644 --- a/celery_haystack/signals.py +++ b/celery_haystack/signals.py @@ -41,4 +41,4 @@ def enqueue(self, action, instance, sender, **kwargs): if isinstance(index, CelerySearchIndex): if action == 'update' and not index.should_update(instance): continue - enqueue_task(action, instance) + enqueue_task(action, instance, task_path=index.task_path) diff --git a/celery_haystack/utils.py b/celery_haystack/utils.py index 855c66c..f3fc202 100644 --- a/celery_haystack/utils.py +++ b/celery_haystack/utils.py @@ -38,7 +38,8 @@ def enqueue_task(action, instance, **kwargs): if settings.CELERY_HAYSTACK_COUNTDOWN: options['countdown'] = settings.CELERY_HAYSTACK_COUNTDOWN - task = get_update_task() + task_path = kwargs.get('task_path', None) + task = get_update_task(task_path=task_path) def task_func(): return task.apply_async((action, identifier), kwargs, **options) From 2479424f326f97e6752a9c8fee4c14bb74ef3a53 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Thu, 28 Jul 2016 15:19:45 -0400 Subject: [PATCH 2/9] relative imports cause problems whhen you import signals.py to subclass the task handler --- celery_haystack/signals.py | 4 ++-- celery_haystack/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery_haystack/signals.py b/celery_haystack/signals.py index 682221f..f753711 100644 --- a/celery_haystack/signals.py +++ b/celery_haystack/signals.py @@ -3,8 +3,8 @@ from haystack.signals import BaseSignalProcessor from haystack.exceptions import NotHandled -from .utils import enqueue_task -from .indexes import CelerySearchIndex +from celery_haystack.utils import enqueue_task +from celery_haystack.indexes import CelerySearchIndex class CelerySignalProcessor(BaseSignalProcessor): diff --git a/celery_haystack/utils.py b/celery_haystack/utils.py index f3fc202..3389f2a 100644 --- a/celery_haystack/utils.py +++ b/celery_haystack/utils.py @@ -7,7 +7,7 @@ from haystack.utils import get_identifier -from .conf import settings +from celery_haystack.conf import settings def get_update_task(task_path=None): From b85c503d4b82634d9e1b7fe313c60ad9294491ed Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Thu, 28 Jul 2016 15:55:10 -0400 Subject: [PATCH 3/9] Revert "relative imports cause problems whhen you import signals.py to subclass the task handler" This reverts commit 2479424f326f97e6752a9c8fee4c14bb74ef3a53. --- celery_haystack/signals.py | 4 ++-- celery_haystack/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery_haystack/signals.py b/celery_haystack/signals.py index f753711..682221f 100644 --- a/celery_haystack/signals.py +++ b/celery_haystack/signals.py @@ -3,8 +3,8 @@ from haystack.signals import BaseSignalProcessor from haystack.exceptions import NotHandled -from celery_haystack.utils import enqueue_task -from celery_haystack.indexes import CelerySearchIndex +from .utils import enqueue_task +from .indexes import CelerySearchIndex class CelerySignalProcessor(BaseSignalProcessor): diff --git a/celery_haystack/utils.py b/celery_haystack/utils.py index 3389f2a..f3fc202 100644 --- a/celery_haystack/utils.py +++ b/celery_haystack/utils.py @@ -7,7 +7,7 @@ from haystack.utils import get_identifier -from celery_haystack.conf import settings +from .conf import settings def get_update_task(task_path=None): From 48a675b226bd425561bb706fb222f3743b30563a Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Fri, 29 Jul 2016 10:40:52 -0400 Subject: [PATCH 4/9] i think this is what i need to do for task subclassing --- celery_haystack/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery_haystack/__init__.py b/celery_haystack/__init__.py index 2678360..b3f27ac 100644 --- a/celery_haystack/__init__.py +++ b/celery_haystack/__init__.py @@ -1,5 +1,9 @@ __version__ = '0.10' +# this is not a django AppConfig object and therefore needs to be imported here so other tasks can subclass CeleryHaystackSignalHandler +from .conf import CeleryHaystack + + def version_hook(config): config['metadata']['version'] = __version__ From 57b542eaf58d8be46b08459480b4de71bface28c Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Fri, 29 Jul 2016 10:56:45 -0400 Subject: [PATCH 5/9] Revert "i think this is what i need to do for task subclassing" This reverts commit 48a675b226bd425561bb706fb222f3743b30563a. --- celery_haystack/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/celery_haystack/__init__.py b/celery_haystack/__init__.py index b3f27ac..2678360 100644 --- a/celery_haystack/__init__.py +++ b/celery_haystack/__init__.py @@ -1,9 +1,5 @@ __version__ = '0.10' -# this is not a django AppConfig object and therefore needs to be imported here so other tasks can subclass CeleryHaystackSignalHandler -from .conf import CeleryHaystack - - def version_hook(config): config['metadata']['version'] = __version__ From 2fd816d9f073895cad9a8c0fa56951cbd6c743a7 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Fri, 29 Jul 2016 11:17:32 -0400 Subject: [PATCH 6/9] next attempt at getting settings working with a proper app config to make sure this file runs --- celery_haystack/apps.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 celery_haystack/apps.py diff --git a/celery_haystack/apps.py b/celery_haystack/apps.py new file mode 100644 index 0000000..64beabc --- /dev/null +++ b/celery_haystack/apps.py @@ -0,0 +1,6 @@ +from .conf import # this populates our settings +from django.apps import apps as django_apps + + +class CeleryHaystackAppConfig(django_apps.AppConfig): + name = "celery_haystack" From f8532fbe9ea85db116539d9670fd19aed6a14da3 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Fri, 29 Jul 2016 11:19:19 -0400 Subject: [PATCH 7/9] and import the class --- celery_haystack/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery_haystack/apps.py b/celery_haystack/apps.py index 64beabc..877b311 100644 --- a/celery_haystack/apps.py +++ b/celery_haystack/apps.py @@ -1,4 +1,4 @@ -from .conf import # this populates our settings +from .conf import CeleryHaystack # this populates our settings from django.apps import apps as django_apps From d1aacadde8b2d4f04c4941be0d0a8d6a685c5595 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Fri, 29 Jul 2016 11:21:15 -0400 Subject: [PATCH 8/9] I'm terrible at this --- celery_haystack/apps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery_haystack/apps.py b/celery_haystack/apps.py index 877b311..e0031c7 100644 --- a/celery_haystack/apps.py +++ b/celery_haystack/apps.py @@ -1,6 +1,6 @@ from .conf import CeleryHaystack # this populates our settings -from django.apps import apps as django_apps +from django.apps import AppConfig -class CeleryHaystackAppConfig(django_apps.AppConfig): +class CeleryHaystackAppConfig(AppConfig): name = "celery_haystack" From e39b4303afe27a183b187a66ca01cc064e07cbd0 Mon Sep 17 00:00:00 2001 From: John Obelenus Date: Mon, 22 Aug 2016 13:50:35 -0400 Subject: [PATCH 9/9] allow the indexing class to not store results based on a setting --- celery_haystack/conf.py | 2 ++ celery_haystack/tasks.py | 1 + 2 files changed, 3 insertions(+) diff --git a/celery_haystack/conf.py b/celery_haystack/conf.py index 5af93a4..ab76c78 100644 --- a/celery_haystack/conf.py +++ b/celery_haystack/conf.py @@ -20,6 +20,8 @@ class CeleryHaystack(AppConf): QUEUE = None #: Whether the task should be handled transaction safe TRANSACTION_SAFE = True + #: Whether the task shoild ignore results + IGNORE_RESULTS = False #: The batch size used by the CeleryHaystackUpdateIndex task COMMAND_BATCH_SIZE = None diff --git a/celery_haystack/tasks.py b/celery_haystack/tasks.py index 20f798a..1defd8b 100644 --- a/celery_haystack/tasks.py +++ b/celery_haystack/tasks.py @@ -19,6 +19,7 @@ class CeleryHaystackSignalHandler(Task): using = settings.CELERY_HAYSTACK_DEFAULT_ALIAS max_retries = settings.CELERY_HAYSTACK_MAX_RETRIES default_retry_delay = settings.CELERY_HAYSTACK_RETRY_DELAY + ignore_result = settings.CELERY_HAYSTACK_IGNORE_RESULTS def split_identifier(self, identifier, **kwargs): """