Remove arrow dependency, update packages, set up testing with compose (…

…Koed00#17)

.github/workflows/test.yml

@@ -11,8 +11,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
-        django: [ "2.2", "3.2" ]
+        python-version: [ "3.8", "3.9", "3.10" ]
+        django: [ "3.2", "4.1" ]
+
     services:
       disque:
         image: efrecon/disque:1.0-rc1
@@ -52,6 +53,9 @@ jobs:
         - name: Run Tests
           run: |
             poetry run pytest --cov=./django_q --cov-report=xml
+          env:
+            MONGO_HOST: "127.0.0.1"
+            REDIS_HOST: "127.0.0.1"
         - name: Upload to coveralls
           run: |
             python -m pip install coveralls

Dockerfile.dev

@@ -7,34 +7,18 @@ ENV PYTHONUNBUFFERED 1
 # Sets the default shell to bash
 ENV SHELL /bin/bash
 
-# Creates a non-root user
-RUN adduser --disabled-password docker
-
 # Upgrades pip
-RUN pip install --upgrade pip
-
-# Poetry project setup for development
-# Copies poetry requirements files
-COPY --chown=docker Dockerfile.dev requirements.txt* setup.py* ./
-
-RUN pip install -r requirements.txt
-
-RUN pip install pytest pytest-django codecov poetry
+RUN pip install --upgrade pip 
 
-# Clean up
-RUN apt-get autoremove -y \
-    && apt-get clean -y \
-    && rm -rf /var/lib/apt/lists/*
+# Install poetry
+RUN pip install poetry
 
-WORKDIR /home/docker
+WORKDIR /app
 
-# Sets the binaries to path
-ENV PATH="/home/docker/.local/bin:${PATH}"
+COPY . .
 
-# Copy in as non-root user, so permissions match what we need
-COPY --chown=docker:docker . .
+RUN poetry lock --no-update
 
-RUN python setup.py develop
+RUN poetry config virtualenvs.create false
 
-# Applies the container user to be non-root
-USER docker
+RUN poetry install -E testing

README.rst

@@ -19,23 +19,21 @@ Features
 -  Django Admin integration
 -  PaaS compatible with multiple instances
 -  Multi cluster monitor
--  Redis, Disque, IronMQ, SQS, MongoDB or ORM
+-  Redis, IronMQ, SQS, MongoDB or ORM
 -  Rollbar and Sentry support
 
 Requirements
 ~~~~~~~~~~~~
 
--  `Django <https://www.djangoproject.com>`__ > = 2.2
+-  `Django <https://www.djangoproject.com>`__ > = 3.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
--  `Arrow <https://github.com/crsmithdev/arrow>`__
 -  `Blessed <https://github.com/jquast/blessed>`__
 
-Tested with: Python 3.7, 3.8, 3.9, 3.10 Django 2.2.X and 3.2.X
+Tested with: Python 3.7, 3.8, 3.9, 3.10 Django 3.2.X and 4.1.X
 
 Brokers
 ~~~~~~~
 - `Redis <https://django-q2.readthedocs.org/en/latest/brokers.html#redis>`__
-- `Disque <https://django-q2.readthedocs.org/en/latest/brokers.html#disque>`__
 - `IronMQ <https://django-q2.readthedocs.org/en/latest/brokers.html#ironmq>`__
 - `Amazon SQS <https://django-q2.readthedocs.org/en/latest/brokers.html#amazon-sqs>`__
 - `MongoDB <https://django-q2.readthedocs.org/en/latest/brokers.html#mongodb>`__
@@ -173,14 +171,14 @@ Admin page or directly from your code:
 
     # Run a task every 5 minutes, starting at 6 today
     # for 2 hours
-    import arrow
+    from datetime import datetime
 
     schedule('math.hypot',
              3, 4,
              schedule_type=Schedule.MINUTES,
              minutes=5,
              repeats=24,
-             next_run=arrow.utcnow().replace(hour=18, minute=0))
+             next_run=datetime.utcnow().replace(hour=18, minute=0))
 
     # Use a cron expression
     schedule('math.hypot',
@@ -194,45 +192,15 @@ For more info check the `Schedules <https://django-q2.readthedocs.org/en/latest/
 Testing
 ~~~~~~~
 
-To run the tests you will need the following in addition to install requirements:
-
-* `py.test <http://pytest.org/latest/>`__
-* `pytest-django <https://github.com/pytest-dev/pytest-django>`__
-* Disque from https://github.com/antirez/disque.git
-* Redis
-* MongoDB
-
-Or you can use the included Docker Compose file.
-
-The following commands can be used to run the tests:
+Running tests is easy with docker compose, it will also start the necessary databases. Just run:
 
 .. code:: bash
-
-    # Create virtual environment
-    python -m venv venv
-
-    # Install requirements
-    venv/bin/pip install -r requirements.txt
-
-    # Install test dependencies
-    venv/bin/pip install pytest pytest-django
-
-    # Install django-q
-    venv/bin/python setup.py develop
-
-    # Run required services (you need to have docker-compose installed)
-    docker-compose -f test-services-docker-compose.yaml up -d
-
-    # Run tests
-    venv/bin/pytest
-
-    # Stop the services required by tests (when you no longer plan to run tests)
-    docker-compose -f test-services-docker-compose.yaml down
+    docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
 
 Locale
 ~~~~~~
 
-Currently available in English, German and French.
+Currently available in English, German, Turkish, and French.
 Translation pull requests are always welcome.
 
 Todo

django_q/brokers/__init__.py

@@ -173,11 +173,6 @@ def get_broker(list_key: str = Conf.PREFIX) -> Broker:
         m = importlib.import_module(module)
         broker = getattr(m, func)
         return broker(list_key=list_key)
-    # disque
-    elif Conf.DISQUE_NODES:
-        from django_q.brokers import disque
-
-        return disque.Disque(list_key=list_key)
     # Iron MQ
     elif Conf.IRON_MQ:
         from django_q.brokers import ironmq

django_q/cluster.py

@@ -6,13 +6,10 @@
 import socket
 import traceback
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from multiprocessing import Event, Process, Value, current_process
 from time import sleep
 
-# External
-import arrow
-
 # Django
 from django import core, db
 from django.apps.registry import apps
@@ -47,6 +44,8 @@
 from django_q.signing import BadSignature, SignedPackage
 from django_q.status import Stat, Status
 
+from .utils import add_months, add_years
+
 
 class Cluster:
     def __init__(self, broker: Broker = None):
@@ -635,41 +634,34 @@ def scheduler(broker: Broker = None):
                     q_options["hook"] = s.hook
                 # set up the next run time
                 if s.schedule_type != s.ONCE:
-                    next_run = arrow.get(s.next_run)
+                    next_run = s.next_run
                     while True:
                         if s.schedule_type == s.MINUTES:
-                            next_run = next_run.shift(minutes=+(s.minutes or 1))
+                            next_run = next_run + timedelta(minutes=(s.minutes or 1))
                         elif s.schedule_type == s.HOURLY:
-                            next_run = next_run.shift(hours=+1)
+                            next_run = next_run + timedelta(hours=1)
                         elif s.schedule_type == s.DAILY:
-                            next_run = next_run.shift(days=+1)
+                            next_run = next_run + timedelta(days=1)
                         elif s.schedule_type == s.WEEKLY:
-                            next_run = next_run.shift(weeks=+1)
+                            next_run = next_run + timedelta(weeks=1)
                         elif s.schedule_type == s.MONTHLY:
-                            next_run = next_run.shift(months=+1)
+                            next_run = add_months(next_run, 1)
                         elif s.schedule_type == s.QUARTERLY:
-                            next_run = next_run.shift(months=+3)
+                            next_run = add_months(next_run, 3)
                         elif s.schedule_type == s.YEARLY:
-                            next_run = next_run.shift(years=+1)
+                            next_run = add_years(next_run, 1)
                         elif s.schedule_type == s.CRON:
                             if not croniter:
                                 raise ImportError(
                                     _(
                                         "Please install croniter to enable cron expressions"
                                     )
                                 )
-                            next_run = arrow.get(
-                                croniter(s.cron, localtime()).get_next()
-                            )
-                        if Conf.CATCH_UP or next_run > arrow.utcnow():
+                            next_run = croniter(s.cron, localtime()).get_next(datetime)
+                        if Conf.CATCH_UP or next_run > localtime():
                             break
-                    # arrow always returns a tz aware datetime, and we don't want
-                    # this when we explicitly configured django with USE_TZ=False
-                    s.next_run = (
-                        next_run.datetime
-                        if settings.USE_TZ
-                        else next_run.datetime.replace(tzinfo=None)
-                    )
+
+                    s.next_run = next_run
                     s.repeats += -1
                 # send it to the cluster
                 scheduled_broker = broker

django_q/conf.py

@@ -45,15 +45,6 @@ class Conf:
 
     DJANGO_REDIS = conf.get("django_redis", None)
 
-    # Disque broker
-    DISQUE_NODES = conf.get("disque_nodes", None)
-
-    # Optional Authentication
-    DISQUE_AUTH = conf.get("disque_auth", None)
-
-    # Optional Fast acknowledge
-    DISQUE_FASTACK = conf.get("disque_fastack", False)
-
     # IronMQ broker
     IRON_MQ = conf.get("iron_mq", None)
 

django_q/tests/settings.py

@@ -106,11 +106,16 @@
 
 STATIC_URL = "/static/"
 
+REDIS_HOST = os.environ.get("REDIS_HOST", "redis")
+
+MONGO_HOST = os.environ.get("MONGO_HOST", "mongo")
+
+
 # Django Redis
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/0",
+        "LOCATION": f"redis://{REDIS_HOST}:6379/0",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "PARSER_CLASS": "redis.connection.HiredisParser",
@@ -125,4 +130,5 @@
     "testing": True,
     "log_level": "DEBUG",
     "django_redis": "default",
+    "redis": f"redis://{REDIS_HOST}:6379/0"
 }