Use huey instead of celery.

compose/django/Dockerfile-dev

@@ -1,45 +1,31 @@
-#FROM python:3.6-alpine
+FROM python:3.6-alpine
-FROM python:3.6
 
 ENV PYTHONUNBUFFERED 1
 
-#RUN apk add --no-cache libev-dev git postgresql-dev gcc python3-dev musl-dev
+RUN apk add --no-cache  git build-base gcc \
+    python3-dev postgresql-dev musl-dev \
+    jpeg-dev zlib-dev openjpeg-dev tiff-dev libffi-dev \
+    freetype-dev libev-dev lcms2-dev tk-dev tcl-dev \
+    harfbuzz-dev fribidi-dev libxslt-dev
 
-RUN apt update && apt install -y libev-dev libmagic-dev && pip install bjoern
-
-# Requirements have to be pulled and installed here, otherwise caching won't work
 COPY ./requirements /requirements
 
-
 RUN pip install --cache-dir ./cache/pip -r /requirements/local.txt \
-    && groupadd -r django \
+    && addgroup -g 1000 -S django \
-    && useradd -r -g django django
+    && adduser -u 1000 -S django -G django
 
 COPY . /app
 
-RUN mkdir -p /var/log/gunicorn/ \
+RUN mkdir -p /data/sockets 
-    && mkdir -p /data/sockets \
+#    && chown -R django /app
-    && chown -R django /app \
-    && chown -R root:django /var/log/gunicorn/ \
-    && chmod -R 770 /var/log/gunicorn/ 
 
-COPY ./compose/django/dev-gunicorn-mhackspace.sh /dev-gunicorn-mhackspace.sh
-COPY ./compose/django/live-gunicorn-mhackspace.sh /live-gunicorn-mhackspace.sh
-COPY ./compose/django/stage-gunicorn-mhackspace.sh /stage-gunicorn-mhackspace.sh
 COPY ./compose/django/bjoern.py /bjoern.py
 COPY ./compose/django/entrypoint.sh /entrypoint.sh
+
+
 RUN sed -i 's/\r//' /entrypoint.sh \
-    && sed -i 's/\r//' /stage-gunicorn-mhackspace.sh \
-    && sed -i 's/\r//' /live-gunicorn-mhackspace.sh \
-    && sed -i 's/\r//' /dev-gunicorn-mhackspace.sh \
     && chmod +x /entrypoint.sh \
     && chown django /entrypoint.sh \
-    && chmod +x /dev-gunicorn-mhackspace.sh \
-    && chown django /dev-gunicorn-mhackspace.sh \
-    && chmod +x /stage-gunicorn-mhackspace.sh \
-    && chown django /stage-gunicorn-mhackspace.sh \
-    && chmod +x /live-gunicorn-mhackspace.sh \
-    && chown django /live-gunicorn-mhackspace.sh \
     && chown django /data/sockets 
 
 WORKDIR /app

compose/django/entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 cmd="$@"
 

config/settings/common.py

@@ -414,21 +414,8 @@ MAX_IMAGE_UPLOAD_SIZE = 10485760  # 10MB
 # SLUGLIFIER
 AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
 
-# CELERY
-CELERY_BROKER_URL = env("CELERY_BROKER_URL", default="redis://redis:6379/0")
-CELERY_RESULT_BACKEND = "django-db"
-CELERY_IGNORE_RESULT = False
-CELERY_REDIS_HOST = "redis"
-CELERY_REDIS_PORT = 6379
-CELERY_REDIS_DB = 0
 
-CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
+INSTALLED_APPS += ('huey.contrib.djhuey',)
-INSTALLED_APPS += ("django_celery_results", "django_celery_beat")
-CELERY_TIMEZONE = "UTC"
-CELERY_ENABLE_UTC = True
-CELERY_TASK_SERIALIZER = "json"
-CELERY_RESULT_SERIALIZER = "json"
-# END CELERY
 
 # django-compressor
 # ------------------------------------------------------------------------------
@@ -629,3 +616,38 @@ ST_BASE_DIR = os.path.dirname(__file__)
 #   ST is Spirit forum software config
 
 RFID_SECRET = env("RFID_SECRET")
+
+HUEY = {
+    'name': DATABASES['default']['NAME'],  # Use db name for huey.
+    'result_store': True,  # Store return values of tasks.
+    'events': True,  # Consumer emits events allowing real-time monitoring.
+    'store_none': False,  # If a task returns None, do not save to results.
+    'always_eager': DEBUG,  # If DEBUG=True, run synchronously.
+    'store_errors': True,  # Store error info if task throws exception.
+    'blocking': False,  # Poll the queue rather than do blocking pop.
+    'backend_class': 'huey.RedisHuey',  # Use path to redis huey by default,
+    'connection': {
+        'host': 'redis',
+        'port': 6379,
+        'db': 0,
+        'connection_pool': None,  # Definitely you should use pooling!
+        # ... tons of other options, see redis-py for details.
+
+        # huey-specific connection parameters.
+        'read_timeout': 1,  # If not polling (blocking pop), use timeout.
+        'max_errors': 1000,  # Only store the 1000 most recent errors.
+        'url': None,  # Allow Redis config via a DSN.
+    },
+    'consumer': {
+        'workers': 1,
+        'worker_type': 'thread',
+        'initial_delay': 0.1,  # Smallest polling interval, same as -d.
+        'backoff': 1.15,  # Exponential backoff using this rate, -b.
+        'max_delay': 10.0,  # Max possible polling interval, -m.
+        'utc': True,  # Treat ETAs and schedules as UTC datetimes.
+        'scheduler_interval': 1,  # Check schedule every second, -s.
+        'periodic': True,  # Enable crontab feature.
+        'check_worker_health': True,  # Enable worker health checks.
+        'health_check_interval': 1,  # Check worker health every second.
+    },
+}

config/settings/local.py

@@ -111,3 +111,5 @@ COMPRESS_STORAGE = STATICFILES_STORAGE
 DEBUG_TOOLBAR_CONFIG = {
     'INTERCEPT_REDIRECTS': False,
 }
+
+

live.yml

@@ -35,31 +35,21 @@ services:
     image: redis:latest
     restart: always
 
-  celeryworker:
-    build:
-      context: .
-      dockerfile: ./compose/django/Dockerfile
-    env_file: .env
-    volumes:
-      - .:/app
-    depends_on:
-      - postgres
-      - redis
-    command: celery -A mhackspace.celeryapp worker -l INFO
-    restart: always
 
-  celerybeat:
+  huey:
     build:
       context: .
-      dockerfile: ./compose/django/Dockerfile
+      dockerfile: ./compose/django/Dockerfile-dev
-    env_file: .env
+    user: django
-    volumes:
-      - .:/app
     depends_on:
       - postgres
       - redis
-    command: celery -A mhackspace.celeryapp beat -l INFO
+    command: python manage.py run_huey
-    restart: always
+    env_file: .env
+    volumes:
+      - .:/app
+      - sockets:/data/sockets
+
   directory:
     image: osixia/openldap:1.2.0
     env_file: .env

local.yml

@@ -26,6 +26,20 @@ services:
       - postgres_data:/var/lib/postgresql/data
     env_file: .env
 
+  huey:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile-dev
+    user: django
+    depends_on:
+      - postgres
+      - redis
+    command: python manage.py run_huey
+    env_file: .env
+    volumes:
+      - .:/app
+      - sockets:/data/sockets
+
   django:
     build:
       context: .
@@ -53,19 +67,6 @@ services:
       - .:/app
       - sockets:/data/sockets
 
-  django_gunicorn:
-    build:
-      context: .
-      dockerfile: ./compose/django/Dockerfile-dev
-    user: django
-    depends_on:
-      - postgres
-    command: /usr/local/bin/gunicorn config.wsgi -w 1 -b unix:/data/sockets/gunicorn-mhackspace.sock --reload --chdir=/app
-    env_file: .env
-    volumes:
-      - .:/app
-      - sockets:/data/sockets
-
   mailhog:
     image: mailhog/mailhog
     ports:
@@ -74,29 +75,6 @@ services:
   redis:
     image: redis:latest
 
-  celeryworker:
-    build:
-      context: .
-      dockerfile: ./compose/django/Dockerfile-dev
-    env_file: .env
-    volumes:
-      - .:/app
-    depends_on:
-      - postgres
-      - redis
-    command: celery -A mhackspace.celeryapp worker -l INFO
-
-  celerybeat:
-    build:
-      context: .
-      dockerfile: ./compose/django/Dockerfile-dev
-    env_file: .env
-    volumes:
-      - .:/app
-    depends_on:
-      - postgres
-      - redis
-    command: celery -A mhackspace.celeryapp beat -l INFO
 
   # port 9000 by default
   bucket:

mhackspace/base/__init__.py

@@ -1,3 +0,0 @@
-from mhackspace.celeryapp import app as celery_app
-
-__all__ = ['celery_app']

mhackspace/base/tasks.py

@@ -1,10 +1,10 @@
 import requests
-from celery import shared_task
+from huey.contrib.djhuey import periodic_task, task
 from django.conf import settings
 from mhackspace.feeds.helper import import_feeds
 
 
-@shared_task
+@task()
 def update_homepage_feeds():
     import_feeds()
     return {'result': 'Homepage feeds imported'}
@@ -16,7 +16,7 @@ matrix_join_room_id_url = matrix_url + "rooms/%21{room}/join?access_token={acces
 matrix_send_msg_url = matrix_url + "/rooms/%21{room}/send/m.room.message?access_token={access_token}"
 
 
-@shared_task
+@task()
 def send_email(email_to,
                message,
                email_from='no-reply@maidstone-hackspace.org.uk',
@@ -33,7 +33,7 @@ def send_email(email_to,
     email.send()
     return {'result', 'Email sent to %s' % email_to}
 
-@shared_task
+@task()
 def matrix_message(message, prefix='', room='default'):
     # we dont rely on theses, so ignore if it goes wrong
     # TODO at least log that something has gone wrong
@@ -67,7 +67,7 @@ def matrix_message(message, prefix='', room='default'):
     return {'result', 'Matrix message sent successfully'}
 
 
-@shared_task
+@task()
 def twitter_message(message, prefix=''):
     import twitter
     api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,

mhackspace/celeryapp.py

@@ -1,19 +0,0 @@
-# from __future__ import absolute_import
-import os
-from django.conf import settings
-from celery import Celery
-
-
-if not settings.configured:
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
-
-app = Celery('mhackspace')
-app.config_from_object('django.conf:settings', namespace='CELERY')
-
-# app.autodiscover_tasks(lambda: ['mhackspace.base'], related_name='tasks')
-app.autodiscover_tasks()
-
-
-@app.task(bind=True)
-def debug_task(self):
-    print('Request: {0!r}'.format(self.request))  # pragma: no cover

mhackspace/feeds/admin.py

@@ -31,7 +31,7 @@ class ArticleAdmin(ModelAdmin):
         return my_urls + urls
 
     def import_articles(self, request):
-        update_homepage_feeds.delay()
+        update_homepage_feeds()#.delay()
         self.message_user(
             request,
             "Importing articles in background refresh in a few minutes",

mhackspace/ldapsync/__init__.py

@@ -1,3 +0,0 @@
-from mhackspace.celeryapp import app as celery_app
-
-__all__ = ['celery_app']

mhackspace/ldapsync/management/commands.py

@@ -7,7 +7,7 @@ from mhackspace.subscriptions.helper import create_or_update_membership
 
 from django.contrib.auth.models import Group
 #from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL
-from celery import shared_task
+from huey.contrib.djhuey import periodic_task, task
 from .models import User
 
 

mhackspace/ldapsync/tasks.py

@@ -1,7 +1,8 @@
 from django.conf import settings
 from django.contrib.auth.models import Group
+from huey.contrib.djhuey import periodic_task, task
+
 from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL
-from celery import shared_task
 import json
 
 
@@ -38,7 +39,7 @@ def ldap_list_users(connection):
 
 
 
-@shared_task
+@task()
 def ldap_add_organizational_unit(connection, name):
     exists = connection.search(
         'cn=%s, %s' % (name, settings.LDAP_ROOT),
@@ -51,7 +52,7 @@ def ldap_add_organizational_unit(connection, name):
     return connection.result
 
 
-@shared_task
+@task()
 def ldap_add_group(connection, group, users):
     exists = connection.search(
         'cn=%s, ou=groups, %s' % (group, settings.LDAP_ROOT),
@@ -67,7 +68,7 @@ def ldap_add_group(connection, group, users):
 
 
 
-@shared_task
+@task()
 def ldap_add_user(connection, username, name='', password=None):
     u = {'objectClass':  ['inetOrgPerson', 'person', 'top'], 'sn': 'user_sn', 'cn': 'First Last', 'userPassword': ''}
     if not password:
@@ -90,7 +91,7 @@ def ldap_add_user(connection, username, name='', password=None):
     return connection.result
 
 
-@shared_task
+@task()
 def complete_directory_sync(self):
     server = Server(settings.LDAP_SERVER)
     conn = Connection(

mhackspace/users/__init__.py

@@ -1,3 +0,0 @@
-from mhackspace.celeryapp import app as celery_app
-
-__all__ = ['celery_app']

mhackspace/users/tasks.py

@@ -1,7 +1,7 @@
-from celery import shared_task
+from huey.contrib.djhuey import periodic_task, task
 from mhackspace.subscriptions.management.commands.update_membership_status import update_subscriptions
 
 
-@shared_task
+@task()
 def update_users_memebership_status():
     update_subscriptions(provider_name='gocardless')

requirements/base.txt

@@ -11,7 +11,7 @@ django-environ==0.4.5
 whitenoise==4.1.2
 # Static and Media Storage
 # ------------------------------------------------
-boto3==1.9.53
+boto3==1.9.96
 django-storages==1.7.1
 # django-storages-redux==1.3.2
 
@@ -20,7 +20,7 @@ django-storages==1.7.1
 django-braces==1.13.0
 django-crispy-forms==1.7.2
 
-django-extensions==2.1.4
+django-extensions==2.1.5
 Werkzeug==0.14.1
 
 # Models
@@ -28,7 +28,7 @@ django-stdimage==4.0.1
 django-model-utils==3.1.2
 
 # Images
-Pillow==5.3.0
+Pillow==5.4.1
 
 # For user registration, either via email or social
 # Well-built with regular release cycles!
@@ -36,13 +36,13 @@ django-allauth==0.38.0
 
 
 # Python-PostgreSQL Database Adapter
-psycopg2==2.7.6.1
+psycopg2==2.7.7
 
 # Unicode slugification
 awesome-slugify==1.6.5
 
 # Time zones support
-pytz==2018.7
+pytz==2018.9
 
 # Redis support
 django-redis==4.10.0
@@ -54,48 +54,46 @@ django-compressor==2.2
 #fix for use with s3 buckets merged in master, so next release we can remove this
 #django-sass-processor==0.5.7
 git+https://github.com/jrief/django-sass-processor.git
-libsass==0.16.1
+libsass==0.17.0
-lxml==4.2.5
+lxml==4.3.1
 
 # WSGI Handler
 # ------------------------------------------------
-gevent==1.3.7
+gevent==1.4.0
 gunicorn==19.9.0
 #https://github.com/jonashaag/bjoern
-#bjoern
+bjoern
 
 # Your custom requirements go here
 mock==2.0.0
 
 gocardless_pro==1.9.0
-braintree==3.49.0
+braintree==3.51.0
 
 django-autofixture==0.12.1
-wiki==0.4.2
+wiki==0.4.4
 
-djangorestframework==3.9.0
+djangorestframework==3.9.1
 djangorestframework-jwt==1.11.0
-django-filter==2.0.0
+django-filter==2.1.0
 coreapi==2.3.3
 # api libraries end
 
-martor==1.3.5
+martor==1.3.8
 
-django-spirit==0.6.2
+django-spirit==0.7.0
-django-djconfig==0.8.0
+django-djconfig==0.9.0
 django-haystack==2.8.1
 django-xforwardedfor-middleware==2.0
 
-# Application queue celery
+# Application queue 
-celery==4.2.1
+huey[backends]
-django-celery-results==1.0.4
-django-celery-beat==1.3.0
 
-argon2-cffi==18.3.0
+argon2-cffi==19.1.0
 django-cors-headers==2.4.0
 python-magic==0.4.15
-ldap3==2.5.1
+ldap3==2.5.2
-bcrypt==3.1.4
+bcrypt==3.1.6
 python-twitter==3.5
 feedparser==5.2.1
-PyJWT==1.6.4
+PyJWT==1.7.1

stage.yml

@@ -36,31 +36,19 @@ services:
     image: redis:latest
     restart: always
 
-  celeryworker:
+  huey:
     build:
       context: .
-      dockerfile: ./compose/django/Dockerfile
+      dockerfile: ./compose/django/Dockerfile-dev
-    env_file: .env
+    user: django
-    volumes:
-      - .:/app
     depends_on:
       - postgres
       - redis
-    command: celery -A mhackspace.celeryapp worker -l INFO
+    command: python manage.py run_huey
-    restart: always
-
-  celerybeat:
-    build:
-      context: .
-      dockerfile: ./compose/django/Dockerfile
     env_file: .env
     volumes:
       - .:/app
-    depends_on:
+      - sockets:/data/sockets
-      - postgres
-      - redis
-    command: celery -A mhackspace.celeryapp beat -l INFO
-    restart: always
 
   directory:
     image: osixia/openldap:1.2.0