oly
/
hackspace
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use huey instead of celery.

This commit is contained in:
Oliver Marks 2019-02-23 22:20:27 +00:00
parent 899fe700b8
commit a3f4efc2c1
17 changed files with 116 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
compose/django/Dockerfile-dev
View File

@ -1,45 +1,31 @@
#FROM python:3.6-alpine
FROM python:3.6
FROM python:3.6-alpine
ENV PYTHONUNBUFFERED 1
#RUN apk add --no-cache libev-dev git postgresql-dev gcc python3-dev musl-dev
RUN apk add --no-cache git build-base gcc \
python3-dev postgresql-dev musl-dev \
jpeg-dev zlib-dev openjpeg-dev tiff-dev libffi-dev \
freetype-dev libev-dev lcms2-dev tk-dev tcl-dev \
harfbuzz-dev fribidi-dev libxslt-dev
RUN apt update && apt install -y libev-dev libmagic-dev && pip install bjoern
# Requirements have to be pulled and installed here, otherwise caching won't work
COPY ./requirements /requirements
RUN pip install --cache-dir ./cache/pip -r /requirements/local.txt \
&& groupadd -r django \
&& useradd -r -g django django
&& addgroup -g 1000 -S django \
&& adduser -u 1000 -S django -G django
COPY . /app
RUN mkdir -p /var/log/gunicorn/ \
&& mkdir -p /data/sockets \
&& chown -R django /app \
&& chown -R root:django /var/log/gunicorn/ \
&& chmod -R 770 /var/log/gunicorn/
RUN mkdir -p /data/sockets
# && chown -R django /app
COPY ./compose/django/dev-gunicorn-mhackspace.sh /dev-gunicorn-mhackspace.sh
COPY ./compose/django/live-gunicorn-mhackspace.sh /live-gunicorn-mhackspace.sh
COPY ./compose/django/stage-gunicorn-mhackspace.sh /stage-gunicorn-mhackspace.sh
COPY ./compose/django/bjoern.py /bjoern.py
COPY ./compose/django/entrypoint.sh /entrypoint.sh
RUN sed -i 's/\r//' /entrypoint.sh \
&& sed -i 's/\r//' /stage-gunicorn-mhackspace.sh \
&& sed -i 's/\r//' /live-gunicorn-mhackspace.sh \
&& sed -i 's/\r//' /dev-gunicorn-mhackspace.sh \
&& chmod +x /entrypoint.sh \
&& chown django /entrypoint.sh \
&& chmod +x /dev-gunicorn-mhackspace.sh \
&& chown django /dev-gunicorn-mhackspace.sh \
&& chmod +x /stage-gunicorn-mhackspace.sh \
&& chown django /stage-gunicorn-mhackspace.sh \
&& chmod +x /live-gunicorn-mhackspace.sh \
&& chown django /live-gunicorn-mhackspace.sh \
&& chown django /data/sockets
WORKDIR /app

2
compose/django/entrypoint.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
set -e
cmd="$@"

50
config/settings/common.py
View File

@ -414,21 +414,8 @@ MAX_IMAGE_UPLOAD_SIZE = 10485760 # 10MB
# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
# CELERY
CELERY_BROKER_URL = env("CELERY_BROKER_URL", default="redis://redis:6379/0")
CELERY_RESULT_BACKEND = "django-db"
CELERY_IGNORE_RESULT = False
CELERY_REDIS_HOST = "redis"
CELERY_REDIS_PORT = 6379
CELERY_REDIS_DB = 0
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
INSTALLED_APPS += ("django_celery_results", "django_celery_beat")
CELERY_TIMEZONE = "UTC"
CELERY_ENABLE_UTC = True
CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json"
# END CELERY
INSTALLED_APPS += ('huey.contrib.djhuey',)
# django-compressor
# ------------------------------------------------------------------------------
@ -629,3 +616,38 @@ ST_BASE_DIR = os.path.dirname(__file__)
# ST is Spirit forum software config
RFID_SECRET = env("RFID_SECRET")
HUEY = {
'name': DATABASES['default']['NAME'], # Use db name for huey.
'result_store': True, # Store return values of tasks.
'events': True, # Consumer emits events allowing real-time monitoring.
'store_none': False, # If a task returns None, do not save to results.
'always_eager': DEBUG, # If DEBUG=True, run synchronously.
'store_errors': True, # Store error info if task throws exception.
'blocking': False, # Poll the queue rather than do blocking pop.
'backend_class': 'huey.RedisHuey', # Use path to redis huey by default,
'connection': {
'host': 'redis',
'port': 6379,
'db': 0,
'connection_pool': None, # Definitely you should use pooling!
# ... tons of other options, see redis-py for details.
# huey-specific connection parameters.
'read_timeout': 1, # If not polling (blocking pop), use timeout.
'max_errors': 1000, # Only store the 1000 most recent errors.
'url': None, # Allow Redis config via a DSN.
},
'consumer': {
'workers': 1,
'worker_type': 'thread',
'initial_delay': 0.1, # Smallest polling interval, same as -d.
'backoff': 1.15, # Exponential backoff using this rate, -b.
'max_delay': 10.0, # Max possible polling interval, -m.
'utc': True, # Treat ETAs and schedules as UTC datetimes.
'scheduler_interval': 1, # Check schedule every second, -s.
'periodic': True, # Enable crontab feature.
'check_worker_health': True, # Enable worker health checks.
'health_check_interval': 1, # Check worker health every second.
},
}

2
config/settings/local.py
View File

@ -111,3 +111,5 @@ COMPRESS_STORAGE = STATICFILES_STORAGE
DEBUG_TOOLBAR_CONFIG = {
'INTERCEPT_REDIRECTS': False,
}

28
live.yml
View File

@ -35,31 +35,21 @@ services:
image: redis:latest
restart: always
celeryworker:
build:
context: .
dockerfile: ./compose/django/Dockerfile
env_file: .env
volumes:
- .:/app
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp worker -l INFO
restart: always
celerybeat:
huey:
build:
context: .
dockerfile: ./compose/django/Dockerfile
env_file: .env
volumes:
- .:/app
dockerfile: ./compose/django/Dockerfile-dev
user: django
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp beat -l INFO
restart: always
command: python manage.py run_huey
env_file: .env
volumes:
- .:/app
- sockets:/data/sockets
directory:
image: osixia/openldap:1.2.0
env_file: .env

50
local.yml
View File

@ -26,6 +26,20 @@ services:
- postgres_data:/var/lib/postgresql/data
env_file: .env
huey:
build:
context: .
dockerfile: ./compose/django/Dockerfile-dev
user: django
depends_on:
- postgres
- redis
command: python manage.py run_huey
env_file: .env
volumes:
- .:/app
- sockets:/data/sockets
django:
build:
context: .
@ -53,19 +67,6 @@ services:
- .:/app
- sockets:/data/sockets
django_gunicorn:
build:
context: .
dockerfile: ./compose/django/Dockerfile-dev
user: django
depends_on:
- postgres
command: /usr/local/bin/gunicorn config.wsgi -w 1 -b unix:/data/sockets/gunicorn-mhackspace.sock --reload --chdir=/app
env_file: .env
volumes:
- .:/app
- sockets:/data/sockets
mailhog:
image: mailhog/mailhog
ports:
@ -74,29 +75,6 @@ services:
redis:
image: redis:latest
celeryworker:
build:
context: .
dockerfile: ./compose/django/Dockerfile-dev
env_file: .env
volumes:
- .:/app
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp worker -l INFO
celerybeat:
build:
context: .
dockerfile: ./compose/django/Dockerfile-dev
env_file: .env
volumes:
- .:/app
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp beat -l INFO
# port 9000 by default
bucket:

3
mhackspace/base/__init__.py
View File

@ -1,3 +0,0 @@
from mhackspace.celeryapp import app as celery_app
__all__ = ['celery_app']

10
mhackspace/base/tasks.py
View File

@ -1,10 +1,10 @@
import requests
from celery import shared_task
from huey.contrib.djhuey import periodic_task, task
from django.conf import settings
from mhackspace.feeds.helper import import_feeds
@shared_task
@task()
def update_homepage_feeds():
import_feeds()
return {'result': 'Homepage feeds imported'}
@ -16,7 +16,7 @@ matrix_join_room_id_url = matrix_url + "rooms/%21{room}/join?access_token={acces
matrix_send_msg_url = matrix_url + "/rooms/%21{room}/send/m.room.message?access_token={access_token}"
@shared_task
@task()
def send_email(email_to,
message,
email_from='no-reply@maidstone-hackspace.org.uk',
@ -33,7 +33,7 @@ def send_email(email_to,
email.send()
return {'result', 'Email sent to %s' % email_to}
@shared_task
@task()
def matrix_message(message, prefix='', room='default'):
# we dont rely on theses, so ignore if it goes wrong
# TODO at least log that something has gone wrong
@ -67,7 +67,7 @@ def matrix_message(message, prefix='', room='default'):
return {'result', 'Matrix message sent successfully'}
@shared_task
@task()
def twitter_message(message, prefix=''):
import twitter
api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,

19
mhackspace/celeryapp.py
View File

@ -1,19 +0,0 @@
# from __future__ import absolute_import
import os
from django.conf import settings
from celery import Celery
if not settings.configured:
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
app = Celery('mhackspace')
app.config_from_object('django.conf:settings', namespace='CELERY')
# app.autodiscover_tasks(lambda: ['mhackspace.base'], related_name='tasks')
app.autodiscover_tasks()
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request)) # pragma: no cover

2
mhackspace/feeds/admin.py
View File

@ -31,7 +31,7 @@ class ArticleAdmin(ModelAdmin):
return my_urls + urls
def import_articles(self, request):
update_homepage_feeds.delay()
update_homepage_feeds()#.delay()
self.message_user(
request,
"Importing articles in background refresh in a few minutes",

3
mhackspace/ldapsync/__init__.py
View File

@ -1,3 +0,0 @@
from mhackspace.celeryapp import app as celery_app
__all__ = ['celery_app']

2
mhackspace/ldapsync/management/commands.py
View File

@ -7,7 +7,7 @@ from mhackspace.subscriptions.helper import create_or_update_membership
from django.contrib.auth.models import Group
#from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL
from celery import shared_task
from huey.contrib.djhuey import periodic_task, task
from .models import User

11
mhackspace/ldapsync/tasks.py
View File

@ -1,7 +1,8 @@
from django.conf import settings
from django.contrib.auth.models import Group
from huey.contrib.djhuey import periodic_task, task
from ldap3 import Server, Connection, ObjectDef, AttrDef, Reader, Writer, ALL
from celery import shared_task
import json
@ -38,7 +39,7 @@ def ldap_list_users(connection):
@shared_task
@task()
def ldap_add_organizational_unit(connection, name):
exists = connection.search(
'cn=%s, %s' % (name, settings.LDAP_ROOT),
@ -51,7 +52,7 @@ def ldap_add_organizational_unit(connection, name):
return connection.result
@shared_task
@task()
def ldap_add_group(connection, group, users):
exists = connection.search(
'cn=%s, ou=groups, %s' % (group, settings.LDAP_ROOT),
@ -67,7 +68,7 @@ def ldap_add_group(connection, group, users):
@shared_task
@task()
def ldap_add_user(connection, username, name='', password=None):
u = {'objectClass': ['inetOrgPerson', 'person', 'top'], 'sn': 'user_sn', 'cn': 'First Last', 'userPassword': ''}
if not password:
@ -90,7 +91,7 @@ def ldap_add_user(connection, username, name='', password=None):
return connection.result
@shared_task
@task()
def complete_directory_sync(self):
server = Server(settings.LDAP_SERVER)
conn = Connection(

3
mhackspace/users/__init__.py
View File

@ -1,3 +0,0 @@
from mhackspace.celeryapp import app as celery_app
__all__ = ['celery_app']

4
mhackspace/users/tasks.py
View File

@ -1,7 +1,7 @@
from celery import shared_task
from huey.contrib.djhuey import periodic_task, task
from mhackspace.subscriptions.management.commands.update_membership_status import update_subscriptions
@shared_task
@task()
def update_users_memebership_status():
update_subscriptions(provider_name='gocardless')

46
requirements/base.txt
View File

@ -11,7 +11,7 @@ django-environ==0.4.5
whitenoise==4.1.2
# Static and Media Storage
# ------------------------------------------------
boto3==1.9.53
boto3==1.9.96
django-storages==1.7.1
# django-storages-redux==1.3.2
@ -20,7 +20,7 @@ django-storages==1.7.1
django-braces==1.13.0
django-crispy-forms==1.7.2
django-extensions==2.1.4
django-extensions==2.1.5
Werkzeug==0.14.1
# Models
@ -28,7 +28,7 @@ django-stdimage==4.0.1
django-model-utils==3.1.2
# Images
Pillow==5.3.0
Pillow==5.4.1
# For user registration, either via email or social
# Well-built with regular release cycles!
@ -36,13 +36,13 @@ django-allauth==0.38.0
# Python-PostgreSQL Database Adapter
psycopg2==2.7.6.1
psycopg2==2.7.7
# Unicode slugification
awesome-slugify==1.6.5
# Time zones support
pytz==2018.7
pytz==2018.9
# Redis support
django-redis==4.10.0
@ -54,48 +54,46 @@ django-compressor==2.2
#fix for use with s3 buckets merged in master, so next release we can remove this
#django-sass-processor==0.5.7
git+https://github.com/jrief/django-sass-processor.git
libsass==0.16.1
lxml==4.2.5
libsass==0.17.0
lxml==4.3.1
# WSGI Handler
# ------------------------------------------------
gevent==1.3.7
gevent==1.4.0
gunicorn==19.9.0
#https://github.com/jonashaag/bjoern
#bjoern
bjoern
# Your custom requirements go here
mock==2.0.0
gocardless_pro==1.9.0
braintree==3.49.0
braintree==3.51.0
django-autofixture==0.12.1
wiki==0.4.2
wiki==0.4.4
djangorestframework==3.9.0
djangorestframework==3.9.1
djangorestframework-jwt==1.11.0
django-filter==2.0.0
django-filter==2.1.0
coreapi==2.3.3
# api libraries end
martor==1.3.5
martor==1.3.8
django-spirit==0.6.2
django-djconfig==0.8.0
django-spirit==0.7.0
django-djconfig==0.9.0
django-haystack==2.8.1
django-xforwardedfor-middleware==2.0
# Application queue celery
celery==4.2.1
django-celery-results==1.0.4
django-celery-beat==1.3.0
# Application queue
huey[backends]
argon2-cffi==18.3.0
argon2-cffi==19.1.0
django-cors-headers==2.4.0
python-magic==0.4.15
ldap3==2.5.1
bcrypt==3.1.4
ldap3==2.5.2
bcrypt==3.1.6
python-twitter==3.5
feedparser==5.2.1
PyJWT==1.6.4
PyJWT==1.7.1

22
stage.yml
View File

@ -36,31 +36,19 @@ services:
image: redis:latest
restart: always
celeryworker:
huey:
build:
context: .
dockerfile: ./compose/django/Dockerfile
env_file: .env
volumes:
- .:/app
dockerfile: ./compose/django/Dockerfile-dev
user: django
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp worker -l INFO
restart: always
celerybeat:
build:
context: .
dockerfile: ./compose/django/Dockerfile
command: python manage.py run_huey
env_file: .env
volumes:
- .:/app
depends_on:
- postgres
- redis
command: celery -A mhackspace.celeryapp beat -l INFO
restart: always
- sockets:/data/sockets
directory:
image: osixia/openldap:1.2.0