From 05d0663ef98fb758fe98105c3c92330012cfddf4 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 15 Feb 2016 15:16:15 +0100 Subject: [PATCH 01/10] Updates botocore to the latest version for testing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6515851..254d9ead 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ arrow==0.7.0 blessed==1.14.1 boto3==1.2.3 -botocore==1.3.23 # via boto3 +botocore==1.3.26 # via boto3 django-picklefield==0.3.2 django-redis==4.3.0 docutils==0.12 # via botocore From c66f0b74edf391a146f39f84ecf99c1a6e204443 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Tue, 16 Feb 2016 11:29:43 +0100 Subject: [PATCH 02/10] Closes all db connections on worker recycle --- django_q/cluster.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_q/cluster.py b/django_q/cluster.py index da7c35a1..0b853ede 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -388,6 +388,7 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT): timer.value = -1 # Idle # Recycle if task_count == Conf.RECYCLE: + db.connections.close_all() # Close any active connections timer.value = -2 # Recycled break logger.info(_('{} stopped doing work').format(name)) From 2e965b1631ef9feced9eee7860b8b41b787f40da Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 24 Feb 2016 10:08:09 +0100 Subject: [PATCH 03/10] Closes db connections on reincarnate --- django_q/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 0b853ede..b3d1c0fe 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -166,6 +166,7 @@ def reincarnate(self, process): :param process: the process to reincarnate :type process: Process or None """ + db.connections.close_all() # Close any old connections if process == self.monitor: self.monitor = self.spawn_monitor() logger.error(_("reincarnated monitor {} after sudden death").format(process.name)) @@ -388,7 +389,6 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT): timer.value = -1 # Idle # Recycle if task_count == Conf.RECYCLE: - db.connections.close_all() # Close any active connections timer.value = -2 # Recycled break logger.info(_('{} stopped doing work').format(name)) From 0a8f2e7e72acf4c674291a9697c2dffc62f842be Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 24 Feb 2016 10:44:47 +0100 Subject: [PATCH 04/10] Adds optional `task_name` argument Makes it possible to name the task yourself. --- django_q/tasks.py | 2 +- docs/tasks.rst | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/django_q/tasks.py b/django_q/tasks.py index 4af17880..5be767cf 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -24,7 +24,7 @@ def async(func, *args, **kwargs): tag = uuid() # build the task package task = {'id': tag[1], - 'name': tag[0], + 'name': q_options.pop('task_name', None) or tag[0], 'func': func, 'args': args} # push optionals diff --git a/docs/tasks.rst b/docs/tasks.rst index 3fe13734..3bd6eb49 100644 --- a/docs/tasks.rst +++ b/docs/tasks.rst @@ -69,6 +69,11 @@ broker """""" A broker instance, in case you want to control your own connections. +task_name +""""""""" + +Optionally overwrites the auto-generated task name. + q_options """"""""" None of the option keywords get passed on to the task function. From 397503f5b22c94739113a3a51c3c099921e8aa9d Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 24 Feb 2016 10:52:25 +0100 Subject: [PATCH 05/10] Adds optional `task_name` argument --- django_q/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tasks.py b/django_q/tasks.py index 5be767cf..842ed9ed 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -24,7 +24,7 @@ def async(func, *args, **kwargs): tag = uuid() # build the task package task = {'id': tag[1], - 'name': q_options.pop('task_name', None) or tag[0], + 'name': keywords.pop('task_name', None) or tag[0], 'func': func, 'args': args} # push optionals From 763fff34c0e271f9133545bbf75c4090a9d3fff7 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 24 Feb 2016 11:22:58 +0100 Subject: [PATCH 06/10] Makes it possible to set a schedules group name via q_options dict --- django_q/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index b3d1c0fe..e88abf37 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -530,7 +530,7 @@ def scheduler(broker=None): s.repeats += -1 # send it to the cluster q_options['broker'] = broker - q_options['group'] = s.name or s.id + q_options['group'] = q_options.get('group', s.name or s.id) kwargs['q_options'] = q_options s.task = tasks.async(s.func, *args, **kwargs) # log it From 8f270a5b978c11a736494cfe91cb35767b816ca3 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 24 Feb 2016 11:27:16 +0100 Subject: [PATCH 07/10] Allows for blank Schedule names in the admin --- .../migrations/0008_auto_20160224_1026.py | 19 +++++++++++++++++++ django_q/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_q/migrations/0008_auto_20160224_1026.py diff --git a/django_q/migrations/0008_auto_20160224_1026.py b/django_q/migrations/0008_auto_20160224_1026.py new file mode 100644 index 00000000..c9aeef18 --- /dev/null +++ b/django_q/migrations/0008_auto_20160224_1026.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_q', '0007_ormq'), + ] + + operations = [ + migrations.AlterField( + model_name='schedule', + name='name', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/django_q/models.py b/django_q/models.py index ceba5ad6..a4b0223a 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -121,7 +121,7 @@ class Meta: class Schedule(models.Model): - name = models.CharField(max_length=100, null=True) + name = models.CharField(max_length=100, null=True, blank=True) func = models.CharField(max_length=256, help_text='e.g. module.tasks.function') hook = models.CharField(max_length=256, null=True, blank=True, help_text='e.g. module.tasks.result_function') args = models.TextField(null=True, blank=True, help_text=_("e.g. 1, 2, 'John'")) From 1e5509de4564880ab3a598daef2c727c7100a54f Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Feb 2016 15:05:50 +0100 Subject: [PATCH 08/10] Also fetches name override from q_options --- django_q/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tasks.py b/django_q/tasks.py index 842ed9ed..f4e00361 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -24,7 +24,7 @@ def async(func, *args, **kwargs): tag = uuid() # build the task package task = {'id': tag[1], - 'name': keywords.pop('task_name', None) or tag[0], + 'name': keywords.pop('task_name', None) or q_options.pop('task_name', None) or tag[0], 'func': func, 'args': args} # push optionals From d4e0646097e0eced0341308e743f4950bb22a908 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Feb 2016 15:11:48 +0100 Subject: [PATCH 09/10] Also fetches name override from q_options --- django_q/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tasks.py b/django_q/tasks.py index f4e00361..31690c8c 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -19,7 +19,7 @@ def async(func, *args, **kwargs): """Queue a task for the cluster.""" keywords = kwargs.copy() opt_keys = ('hook', 'group', 'save', 'sync', 'cached', 'iter_count', 'iter_cached', 'chain', 'broker') - q_options = keywords.pop('q_options', None) + q_options = keywords.pop('q_options', {}) # get an id tag = uuid() # build the task package From 681177e1ec620a490cda96e084b29c484c5e1355 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Feb 2016 15:22:01 +0100 Subject: [PATCH 10/10] Updates Boto, psutil and rollbar to the latest versions --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 254d9ead..fb6d0d89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ # arrow==0.7.0 blessed==1.14.1 -boto3==1.2.3 -botocore==1.3.26 # via boto3 +boto3==1.2.5 +botocore==1.3.30 # via boto3 django-picklefield==0.3.2 django-redis==4.3.0 docutils==0.12 # via botocore @@ -16,11 +16,11 @@ hiredis==0.2.0 iron-core==1.2.0 # via iron-mq iron-mq==0.8 jmespath==0.9.0 # via boto3, botocore -psutil==3.4.2 +psutil==4.0.0 pymongo==3.2.1 python-dateutil==2.4.2 # via arrow, botocore, iron-core redis==2.10.5 requests==2.9.1 # via iron-core, rollbar -rollbar==0.11.2 +rollbar==0.11.3 six==1.10.0 # via blessed, python-dateutil, rollbar wcwidth==0.1.6 # via blessed