From 633497be6cccd59b4a9c863deda94b28b1ec7fe2 Mon Sep 17 00:00:00 2001 From: Oliver Marks Date: Mon, 14 Aug 2017 19:10:00 +0100 Subject: [PATCH] celery integration --- compose/django/entrypoint.sh | 2 ++ compose/nginx/Dockerfile | 3 ++ config/settings/common.py | 12 ++++++- config/settings/local.py | 5 +++ config/settings/production.py | 2 +- config/settings/stage.py | 2 +- dev.yml | 35 +++++++++++++++---- docker-compose.yml | 4 +-- env.example | 11 ++++-- live.yml | 27 ++++++++++++-- mhackspace/base/__init__.py | 5 +++ mhackspace/base/tasks.py | 16 +++++++++ mhackspace/celeryapp.py | 22 ++++++++++++ mhackspace/contact/__init__.py | 0 mhackspace/members/__init__.py | 0 .../tests/test_payment_gateways.py | 1 - .../migrations/0004_auto_20170813_1557.py | 25 +++++++++++++ requirements/base.txt | 6 +++- stage.yml | 27 ++++++++++++-- 19 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 mhackspace/base/tasks.py create mode 100644 mhackspace/celeryapp.py create mode 100644 mhackspace/contact/__init__.py create mode 100644 mhackspace/members/__init__.py create mode 100644 mhackspace/users/migrations/0004_auto_20170813_1557.py diff --git a/compose/django/entrypoint.sh b/compose/django/entrypoint.sh index 158fe37..8e7d7ed 100644 --- a/compose/django/entrypoint.sh +++ b/compose/django/entrypoint.sh @@ -15,6 +15,8 @@ fi export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:5432/$POSTGRES_USER +export CELERY_BROKER_URL=$REDIS_URL/0 + function postgres_ready(){ python << END diff --git a/compose/nginx/Dockerfile b/compose/nginx/Dockerfile index dd4d6bf..ae892f2 100644 --- a/compose/nginx/Dockerfile +++ b/compose/nginx/Dockerfile @@ -2,6 +2,9 @@ FROM nginx:latest ADD nginx.conf /etc/nginx/nginx.conf +# installs the `ps` command in the nginx image +RUN apt-get update && apt-get install -y procps + ADD start.sh /start.sh #ADD nginx-secure.conf /etc/nginx/nginx-secure.conf #ADD dhparams.pem /etc/ssl/private/dhparams.pem diff --git a/config/settings/common.py b/config/settings/common.py index 5c86d97..f0661ab 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -14,7 +14,6 @@ import os import time import environ - # from spirit.settings import * ROOT_DIR = environ.Path(__file__) - 3 # (mhackspace/config/settings/common.py - 3 = mhackspace/) APPS_DIR = ROOT_DIR.path('mhackspace') @@ -341,6 +340,17 @@ LOGIN_URL = 'account_login' # SLUGLIFIER AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify' + +########## CELERY +CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://') +if CELERY_BROKER_URL == 'django://': + CELERY_RESULT_BACKEND = 'redis://' +else: + CELERY_RESULT_BACKEND = CELERY_BROKER_URL +CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' +INSTALLED_APPS += ('django_celery_results','django_celery_beat',) +########## END CELERY + # django-compressor # ------------------------------------------------------------------------------ INSTALLED_APPS += ("compressor", 'sass_processor',) diff --git a/config/settings/local.py b/config/settings/local.py index 533b8a4..37a23f2 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -83,6 +83,11 @@ INSTALLED_APPS += ('django_extensions', ) # ------------------------------------------------------------------------------ TEST_RUNNER = 'django.test.runner.DiscoverRunner' +########## CELERY +# In development, all tasks will be executed locally by blocking until the task returns +CELERY_ALWAYS_EAGER = True +########## END CELERY + # Your local stuff: Below this line define 3rd party library settings # ------------------------------------------------------------------------------ CAPTCHA = { diff --git a/config/settings/production.py b/config/settings/production.py index abe0ba9..6cd0db0 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -184,7 +184,7 @@ LOGGING = { }, 'handlers': { 'mail_admins': { - 'level': 'ERROR', + 'level': 'DEBUG', 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, diff --git a/config/settings/stage.py b/config/settings/stage.py index 2fff117..6ef74ac 100644 --- a/config/settings/stage.py +++ b/config/settings/stage.py @@ -185,7 +185,7 @@ LOGGING = { }, 'handlers': { 'mail_admins': { - 'level': 'ERROR', + 'level': 'DEBUG', 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, diff --git a/dev.yml b/dev.yml index d998514..2ece4d4 100644 --- a/dev.yml +++ b/dev.yml @@ -10,8 +10,7 @@ services: volumes: - postgres_data_dev:/var/lib/postgresql/data - postgres_backup_dev:/backups - environment: - - POSTGRES_USER=mhackspace + env_file: .env django: build: @@ -21,9 +20,6 @@ services: depends_on: - postgres env_file: .env - # environment: - # - POSTGRES_USER=mhackspace - # - USE_DOCKER=yes volumes: - .:/app ports: @@ -42,5 +38,30 @@ services: command: npm install volumes: - ./:/usr/src/app - # redis: - # image: redis:latest + redis: + image: redis:latest + + celeryworker: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp worker -l INFO + + celerybeat: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp beat -l debug + diff --git a/docker-compose.yml b/docker-compose.yml index 7f73066..f5884c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,6 @@ services: # - TERM=xterm - # redis: - # image: redis:latest + redis: + image: redis:latest diff --git a/env.example b/env.example index b194a95..1697a8e 100644 --- a/env.example +++ b/env.example @@ -1,11 +1,16 @@ +#rename me to .env, and change per environment, .env should not be commited contains +#sensitive settings + +#for dev only +USE_DOCKER_DEBUG=yes # PostgreSQL POSTGRES_PASSWORD=mysecretpass -POSTGRES_USER=postgresuser +POSTGRES_USER=mhackspace # General settings -DJANGO_ADMIN_URL= -DJANGO_SETTINGS_MODULE=config.settings.production +DJANGO_ADMIN_URL=admin +DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=iud%k99yw!e+z+c12uatugbn=&lsdyd(t_byk9)@dp@lj6*c*n DJANGO_ALLOWED_HOSTS=.maidstone-hackspace.org.uk diff --git a/live.yml b/live.yml index 2614706..89c901d 100644 --- a/live.yml +++ b/live.yml @@ -37,6 +37,29 @@ services: volumes: - ./:/usr/src/app - # redis: - # image: redis:latest + redis: + image: redis:latest + celeryworker: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp worker -l INFO + + celerybeat: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp beat -l INFO diff --git a/mhackspace/base/__init__.py b/mhackspace/base/__init__.py index e69de29..8eb049c 100644 --- a/mhackspace/base/__init__.py +++ b/mhackspace/base/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, unicode_literals + +from mhackspace.celeryapp import app as celery_app + +__all__ = ['celery_app'] diff --git a/mhackspace/base/tasks.py b/mhackspace/base/tasks.py new file mode 100644 index 0000000..8328b1f --- /dev/null +++ b/mhackspace/base/tasks.py @@ -0,0 +1,16 @@ +from celery import shared_task + + +@shared_task +def update_homepage_feeds(): + pass + # import_feeds() + +# @task(bind=True) +# @app.on_after_configure.connect +# def setup_periodic_tasks(sender, **kwargs): +# # Executes every Monday morning at 7:30 a.m. +# sender.add_periodic_task( +# crontab(hour=4, minute=0), +# update_homepage_feeds.s(), +# ) diff --git a/mhackspace/celeryapp.py b/mhackspace/celeryapp.py new file mode 100644 index 0000000..5c09831 --- /dev/null +++ b/mhackspace/celeryapp.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import +import os +from celery import Celery +from celery.schedules import crontab +# from django.apps import apps, AppConfig + +# if not settings.configured: + # set the default Django settings module for the 'celery' program. + # ;os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') # pragma: no cover + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') + +from django.conf import settings +app = Celery('mhackspace') +# app.config_from_object(settings) +# app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks(lambda: ['mhackspace.base']) + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) # pragma: no cover diff --git a/mhackspace/contact/__init__.py b/mhackspace/contact/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mhackspace/members/__init__.py b/mhackspace/members/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mhackspace/subscriptions/tests/test_payment_gateways.py b/mhackspace/subscriptions/tests/test_payment_gateways.py index 5b19544..62200d0 100644 --- a/mhackspace/subscriptions/tests/test_payment_gateways.py +++ b/mhackspace/subscriptions/tests/test_payment_gateways.py @@ -43,7 +43,6 @@ class TestPaymentGatewaysGocardless(TestCase): self.assertEqual(result.get('reference'), '01') self.assertEqual(result.get('success'), 'success') - # @patch('mhackspace.subscriptions.payments.gocardless.request.requests.get', autospec=True) @patch('mhackspace.subscriptions.payments.gocardless.client.subscription', autospec=True) @patch('mhackspace.subscriptions.payments.gocardless.client.confirm_resource', autospec=True) def test_confirm_subscription_callback(self, mock_confirm, mock_subscription): diff --git a/mhackspace/users/migrations/0004_auto_20170813_1557.py b/mhackspace/users/migrations/0004_auto_20170813_1557.py new file mode 100644 index 0000000..bd0ca9d --- /dev/null +++ b/mhackspace/users/migrations/0004_auto_20170813_1557.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-08-13 15:57 +from __future__ import unicode_literals + +from django.db import migrations +import stdimage.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0003_merge_20170226_0844'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='image', + ), + migrations.AddField( + model_name='user', + name='_image', + field=stdimage.models.StdImageField(blank=True, db_column='image', null=True, upload_to='avatars/'), + ), + ] diff --git a/requirements/base.txt b/requirements/base.txt index 2e870ec..9d79f8c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -67,5 +67,9 @@ draceditor==1.1.8 django-djconfig django-haystack git+https://github.com/nitely/Spirit.git -# git+https://github.com/olymk2/django-xforwardedfor-middleware.git django-xforwardedfor-middleware==2.0 + +# Application queue celery +celery==4.1 +django-celery-results +django-celery-beat diff --git a/stage.yml b/stage.yml index 588548c..8effba7 100644 --- a/stage.yml +++ b/stage.yml @@ -37,6 +37,29 @@ services: volumes: - ./:/usr/src/app - # redis: - # image: redis:latest + redis: + image: redis:latest + celeryworker: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp worker -l INFO + + celerybeat: + build: + context: . + dockerfile: ./compose/django/Dockerfile + env_file: .env + volumes: + - .:/app + depends_on: + - postgres + - redis + command: celery -A mhackspace.celeryapp beat -l INFO