diff --git a/compose/django/Dockerfile-dev b/compose/django/Dockerfile-dev index 7b2cdfd..7bc1c20 100644 --- a/compose/django/Dockerfile-dev +++ b/compose/django/Dockerfile-dev @@ -1,45 +1,31 @@ -#FROM python:3.6-alpine -FROM python:3.6 +FROM python:3.6-alpine ENV PYTHONUNBUFFERED 1 -#RUN apk add --no-cache libev-dev git postgresql-dev gcc python3-dev musl-dev +RUN apk add --no-cache git build-base gcc \ + python3-dev postgresql-dev musl-dev \ + jpeg-dev zlib-dev openjpeg-dev tiff-dev libffi-dev \ + freetype-dev libev-dev lcms2-dev tk-dev tcl-dev \ + harfbuzz-dev fribidi-dev libxslt-dev -RUN apt update && apt install -y libev-dev libmagic-dev && pip install bjoern - -# Requirements have to be pulled and installed here, otherwise caching won't work COPY ./requirements /requirements - RUN pip install --cache-dir ./cache/pip -r /requirements/local.txt \ - && groupadd -r django \ - && useradd -r -g django django + && addgroup -g 1000 -S django \ + && adduser -u 1000 -S django -G django COPY . /app -RUN mkdir -p /var/log/gunicorn/ \ - && mkdir -p /data/sockets \ - && chown -R django /app \ - && chown -R root:django /var/log/gunicorn/ \ - && chmod -R 770 /var/log/gunicorn/ +RUN mkdir -p /data/sockets +# && chown -R django /app -COPY ./compose/django/dev-gunicorn-mhackspace.sh /dev-gunicorn-mhackspace.sh -COPY ./compose/django/live-gunicorn-mhackspace.sh /live-gunicorn-mhackspace.sh -COPY ./compose/django/stage-gunicorn-mhackspace.sh /stage-gunicorn-mhackspace.sh COPY ./compose/django/bjoern.py /bjoern.py COPY ./compose/django/entrypoint.sh /entrypoint.sh + + RUN sed -i 's/\r//' /entrypoint.sh \ - && sed -i 's/\r//' /stage-gunicorn-mhackspace.sh \ - && sed -i 's/\r//' /live-gunicorn-mhackspace.sh \ - && sed -i 's/\r//' /dev-gunicorn-mhackspace.sh \ && chmod +x /entrypoint.sh \ && chown django /entrypoint.sh \ - && chmod +x /dev-gunicorn-mhackspace.sh \ - && chown django /dev-gunicorn-mhackspace.sh \ - && chmod +x /stage-gunicorn-mhackspace.sh \ - && chown django /stage-gunicorn-mhackspace.sh \ - && chmod +x /live-gunicorn-mhackspace.sh \ - && chown django /live-gunicorn-mhackspace.sh \ && chown django /data/sockets WORKDIR /app diff --git a/compose/django/entrypoint.sh b/compose/django/entrypoint.sh index 8e7d7ed..e13adb7 100644 --- a/compose/django/entrypoint.sh +++ b/compose/django/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e cmd="$@" diff --git a/config/settings/common.py b/config/settings/common.py index 79e695b..95f67b8 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -414,21 +414,8 @@ MAX_IMAGE_UPLOAD_SIZE = 10485760 # 10MB # SLUGLIFIER AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify" -# CELERY -CELERY_BROKER_URL = env("CELERY_BROKER_URL", default="redis://redis:6379/0") -CELERY_RESULT_BACKEND = "django-db" -CELERY_IGNORE_RESULT = False -CELERY_REDIS_HOST = "redis" -CELERY_REDIS_PORT = 6379 -CELERY_REDIS_DB = 0 -CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" -INSTALLED_APPS += ("django_celery_results", "django_celery_beat") -CELERY_TIMEZONE = "UTC" -CELERY_ENABLE_UTC = True -CELERY_TASK_SERIALIZER = "json" -CELERY_RESULT_SERIALIZER = "json" -# END CELERY +INSTALLED_APPS += ('huey.contrib.djhuey',) # django-compressor # ------------------------------------------------------------------------------ @@ -629,3 +616,38 @@ ST_BASE_DIR = os.path.dirname(__file__) # ST is Spirit forum software config RFID_SECRET = env("RFID_SECRET") + +HUEY = { + 'name': DATABASES['default']['NAME'], # Use db name for huey. + 'result_store': True, # Store return values of tasks. + 'events': True, # Consumer emits events allowing real-time monitoring. + 'store_none': False, # If a task returns None, do not save to results. + 'always_eager': DEBUG, # If DEBUG=True, run synchronously. + 'store_errors': True, # Store error info if task throws exception. + 'blocking': False, # Poll the queue rather than do blocking pop. + 'backend_class': 'huey.RedisHuey', # Use path to redis huey by default, + 'connection': { + 'host': 'redis', + 'port': 6379, + 'db': 0, + 'connection_pool': None, # Definitely you should use pooling! + # ... tons of other options, see redis-py for details. + + # huey-specific connection parameters. + 'read_timeout': 1, # If not polling (blocking pop), use timeout. + 'max_errors': 1000, # Only store the 1000 most recent errors. + 'url': None, # Allow Redis config via a DSN. + }, + 'consumer': { + 'workers': 1, + 'worker_type': 'thread', + 'initial_delay': 0.1, # Smallest polling interval, same as -d. + 'backoff': 1.15, # Exponential backoff using this rate, -b. + 'max_delay': 10.0, # Max possible polling interval, -m. + 'utc': True, # Treat ETAs and schedules as UTC datetimes. + 'scheduler_interval': 1, # Check schedule every second, -s. + 'periodic': True, # Enable crontab feature. + 'check_worker_health': True, # Enable worker health checks. + 'health_check_interval': 1, # Check worker health every second. + }, +} diff --git a/config/settings/local.py b/config/settings/local.py index 47e909e..f56f918 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -111,3 +111,5 @@ COMPRESS_STORAGE = STATICFILES_STORAGE DEBUG_TOOLBAR_CONFIG = { 'INTERCEPT_REDIRECTS': False, } + + diff --git a/live.yml b/live.yml index 96a5324..112e515 100644 --- a/live.yml +++ b/live.yml @@ -35,31 +35,21 @@ services: image: redis:latest restart: always - celeryworker: - build: - context: . - dockerfile: ./compose/django/Dockerfile - env_file: .env - volumes: - - .:/app - depends_on: - - postgres - - redis - command: celery -A mhackspace.celeryapp worker -l INFO - restart: always - celerybeat: + huey: build: context: . - dockerfile: ./compose/django/Dockerfile - env_file: .env - volumes: - - .:/app + dockerfile: ./compose/django/Dockerfile-dev + user: django depends_on: - postgres - redis - command: celery -A mhackspace.celeryapp beat -l INFO - restart: always + command: python manage.py run_huey + env_file: .env + volumes: + - .:/app + - sockets:/data/sockets + directory: image: osixia/openldap:1.2.0 env_file: .env diff --git a/local.yml b/local.yml index 93f42b6..32a8637 100644 --- a/local.yml +++ b/local.yml @@ -26,6 +26,20 @@ services: - postgres_data:/var/lib/postgresql/data env_file: .env + huey: + build: + context: . + dockerfile: ./compose/django/Dockerfile-dev + user: django + depends_on: + - postgres + - redis + command: python manage.py run_huey + env_file: .env + volumes: + - .:/app + - sockets:/data/sockets + django: build: context: . @@ -53,19 +67,6 @@ services: - .:/app - sockets:/data/sockets - django_gunicorn: - build: - context: . - dockerfile: ./compose/django/Dockerfile-dev - user: django - depends_on: - - postgres - command: /usr/local/bin/gunicorn config.wsgi -w 1 -b unix:/data/sockets/gunicorn-mhackspace.sock --reload --chdir=/app - env_file: .env - volumes: - - .:/app - - sockets:/data/sockets - mailhog: image: mailhog/mailhog ports: @@ -74,29 +75,6 @@ services: redis: image: redis:latest - celeryworker: - build: - context: . - dockerfile: ./compose/django/Dockerfile-dev - env_file: .env - volumes: - - .:/app - depends_on: - - postgres - - redis - command: celery -A mhackspace.celeryapp worker -l INFO - - celerybeat: - build: - context: . - dockerfile: ./compose/django/Dockerfile-dev - env_file: .env - volumes: - - .:/app - depends_on: - - postgres - - redis - command: celery -A mhackspace.celeryapp beat -l INFO # port 9000 by default bucket: diff --git a/mhackspace/base/__init__.py b/mhackspace/base/__init__.py index 70b88d3..e69de29 100644 --- a/mhackspace/base/__init__.py +++ b/mhackspace/base/__init__.py @@ -1,3 +0,0 @@ -from mhackspace.celeryapp import app as celery_app - -__all__ = ['celery_app'] diff --git a/mhackspace/base/tasks.py b/mhackspace/base/tasks.py index 2b9f64e..03daa3e 100644 --- a/mhackspace/base/tasks.py +++ b/mhackspace/base/tasks.py @@ -1,10 +1,10 @@ import requests -from celery import shared_task +from huey.contrib.djhuey import periodic_task, task from django.conf import settings from mhackspace.feeds.helper import import_feeds -@shared_task +@task() def update_homepage_feeds(): import_feeds() return {'result': 'Homepage feeds imported'} @@ -16,7 +16,7 @@ matrix_join_room_id_url = matrix_url + "rooms/%21{room}/join?access_token={acces matrix_send_msg_url = matrix_url + "/rooms/%21{room}/send/m.room.message?access_token={access_token}" -@shared_task +@task() def send_email(email_to, message, email_from='no-reply@maidstone-hackspace.org.uk', @@ -33,7 +33,7 @@ def send_email(email_to, email.send() return {'result', 'Email sent to %s' % email_to} -@shared_task +@task() def matrix_message(message, prefix='', room='default'): # we dont rely on theses, so ignore if it goes wrong # TODO at least log that something has gone wrong @@ -67,7 +67,7 @@ def matrix_message(message, prefix='', room='default'): return {'result', 'Matrix message sent successfully'} -@shared_task +@task() def twitter_message(message, prefix=''): import twitter api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY, diff --git a/mhackspace/celeryapp.py b/mhackspace/celeryapp.py deleted file mode 100644 index 4315587..0000000 --- a/mhackspace/celeryapp.py +++ /dev/null @@ -1,19 +0,0 @@ -# from __future__ import absolute_import -import os -from django.conf import settings -from celery import Celery - - -if not settings.configured: - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') - -app = Celery('mhackspace') -app.config_from_object('django.conf:settings', namespace='CELERY') - -# app.autodiscover_tasks(lambda: ['mhackspace.base'], related_name='tasks') -app.autodiscover_tasks() - - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) # pragma: no cover diff --git a/mhackspace/feeds/admin.py b/mhackspace/feeds/admin.py index ba08b51..87f5cfa 100644 --- a/mhackspace/feeds/admin.py +++ b/mhackspace/feeds/admin.py @@ -31,7 +31,7 @@ class ArticleAdmin(ModelAdmin): return my_urls + urls def import_articles(self, request): - update_homepage_feeds.delay() + update_homepage_feeds()#.delay() self.message_user( request, "Importing articles in background refresh in a few minutes", diff --git a/mhackspace/ldapsync/__init__.py b/mhackspace/ldapsync/__init__.py index 70b88d3..e69de29 100644 --- a/mhackspace/ldapsync/__init__.py +++ b/mhackspace/ldapsync/__init__.py @@ -1,3 +0,0 @@ -from mhackspace.celeryapp import app as celery_app - -__all__ = ['celery_app'] diff --git a/mhackspace/ldapsync/management/commands.py b/mhackspace/ldapsync/management/commands.py index fb95faa..409ce7f 100644 --- a/mhackspace/ldapsync/management/commands.py +++ b/mhackspace/ldapsync/management/commands.py @@ -7,7 +7,7 @@ from mhackspace.subscriptions.helper import create_or_update_membership from django.contrib.auth.models import Group #from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL -from celery import shared_task +from huey.contrib.djhuey import periodic_task, task from .models import User diff --git a/mhackspace/ldapsync/tasks.py b/mhackspace/ldapsync/tasks.py index 593ec9f..d804f20 100644 --- a/mhackspace/ldapsync/tasks.py +++ b/mhackspace/ldapsync/tasks.py @@ -1,7 +1,8 @@ from django.conf import settings from django.contrib.auth.models import Group +from huey.contrib.djhuey import periodic_task, task + from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL -from celery import shared_task import json @@ -38,7 +39,7 @@ def ldap_list_users(connection): -@shared_task +@task() def ldap_add_organizational_unit(connection, name): exists = connection.search( 'cn=%s, %s' % (name, settings.LDAP_ROOT), @@ -51,7 +52,7 @@ def ldap_add_organizational_unit(connection, name): return connection.result -@shared_task +@task() def ldap_add_group(connection, group, users): exists = connection.search( 'cn=%s, ou=groups, %s' % (group, settings.LDAP_ROOT), @@ -67,7 +68,7 @@ def ldap_add_group(connection, group, users): -@shared_task +@task() def ldap_add_user(connection, username, name='', password=None): u = {'objectClass': ['inetOrgPerson', 'person', 'top'], 'sn': 'user_sn', 'cn': 'First Last', 'userPassword': ''} if not password: @@ -90,7 +91,7 @@ def ldap_add_user(connection, username, name='', password=None): return connection.result -@shared_task +@task() def complete_directory_sync(self): server = Server(settings.LDAP_SERVER) conn = Connection( diff --git a/mhackspace/users/__init__.py b/mhackspace/users/__init__.py index 70b88d3..e69de29 100644 --- a/mhackspace/users/__init__.py +++ b/mhackspace/users/__init__.py @@ -1,3 +0,0 @@ -from mhackspace.celeryapp import app as celery_app - -__all__ = ['celery_app'] diff --git a/mhackspace/users/tasks.py b/mhackspace/users/tasks.py index d733d3e..52a4685 100644 --- a/mhackspace/users/tasks.py +++ b/mhackspace/users/tasks.py @@ -1,7 +1,7 @@ -from celery import shared_task +from huey.contrib.djhuey import periodic_task, task from mhackspace.subscriptions.management.commands.update_membership_status import update_subscriptions -@shared_task +@task() def update_users_memebership_status(): update_subscriptions(provider_name='gocardless') diff --git a/requirements/base.txt b/requirements/base.txt index 36869d6..e06ed59 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -11,7 +11,7 @@ django-environ==0.4.5 whitenoise==4.1.2 # Static and Media Storage # ------------------------------------------------ -boto3==1.9.53 +boto3==1.9.96 django-storages==1.7.1 # django-storages-redux==1.3.2 @@ -20,7 +20,7 @@ django-storages==1.7.1 django-braces==1.13.0 django-crispy-forms==1.7.2 -django-extensions==2.1.4 +django-extensions==2.1.5 Werkzeug==0.14.1 # Models @@ -28,7 +28,7 @@ django-stdimage==4.0.1 django-model-utils==3.1.2 # Images -Pillow==5.3.0 +Pillow==5.4.1 # For user registration, either via email or social # Well-built with regular release cycles! @@ -36,13 +36,13 @@ django-allauth==0.38.0 # Python-PostgreSQL Database Adapter -psycopg2==2.7.6.1 +psycopg2==2.7.7 # Unicode slugification awesome-slugify==1.6.5 # Time zones support -pytz==2018.7 +pytz==2018.9 # Redis support django-redis==4.10.0 @@ -54,48 +54,46 @@ django-compressor==2.2 #fix for use with s3 buckets merged in master, so next release we can remove this #django-sass-processor==0.5.7 git+https://github.com/jrief/django-sass-processor.git -libsass==0.16.1 -lxml==4.2.5 +libsass==0.17.0 +lxml==4.3.1 # WSGI Handler # ------------------------------------------------ -gevent==1.3.7 +gevent==1.4.0 gunicorn==19.9.0 #https://github.com/jonashaag/bjoern -#bjoern +bjoern # Your custom requirements go here mock==2.0.0 gocardless_pro==1.9.0 -braintree==3.49.0 +braintree==3.51.0 django-autofixture==0.12.1 -wiki==0.4.2 +wiki==0.4.4 -djangorestframework==3.9.0 +djangorestframework==3.9.1 djangorestframework-jwt==1.11.0 -django-filter==2.0.0 +django-filter==2.1.0 coreapi==2.3.3 # api libraries end -martor==1.3.5 +martor==1.3.8 -django-spirit==0.6.2 -django-djconfig==0.8.0 +django-spirit==0.7.0 +django-djconfig==0.9.0 django-haystack==2.8.1 django-xforwardedfor-middleware==2.0 -# Application queue celery -celery==4.2.1 -django-celery-results==1.0.4 -django-celery-beat==1.3.0 +# Application queue +huey[backends] -argon2-cffi==18.3.0 +argon2-cffi==19.1.0 django-cors-headers==2.4.0 python-magic==0.4.15 -ldap3==2.5.1 -bcrypt==3.1.4 +ldap3==2.5.2 +bcrypt==3.1.6 python-twitter==3.5 feedparser==5.2.1 -PyJWT==1.6.4 +PyJWT==1.7.1 diff --git a/stage.yml b/stage.yml index 1c15110..20f2a15 100644 --- a/stage.yml +++ b/stage.yml @@ -36,31 +36,19 @@ services: image: redis:latest restart: always - celeryworker: + huey: build: context: . - dockerfile: ./compose/django/Dockerfile - env_file: .env - volumes: - - .:/app + dockerfile: ./compose/django/Dockerfile-dev + user: django depends_on: - postgres - redis - command: celery -A mhackspace.celeryapp worker -l INFO - restart: always - - celerybeat: - build: - context: . - dockerfile: ./compose/django/Dockerfile + command: python manage.py run_huey env_file: .env volumes: - .:/app - depends_on: - - postgres - - redis - command: celery -A mhackspace.celeryapp beat -l INFO - restart: always + - sockets:/data/sockets directory: image: osixia/openldap:1.2.0