celery integration

compose/django/entrypoint.sh

@@ -15,6 +15,8 @@ fi
 
 export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:5432/$POSTGRES_USER
 
+export CELERY_BROKER_URL=$REDIS_URL/0
+
 
 function postgres_ready(){
 python << END

compose/nginx/Dockerfile

@@ -2,6 +2,9 @@ FROM nginx:latest
 ADD nginx.conf /etc/nginx/nginx.conf
 
 
+# installs the `ps` command in the nginx image
+RUN apt-get update && apt-get install -y procps
+
 ADD start.sh /start.sh
 #ADD nginx-secure.conf /etc/nginx/nginx-secure.conf
 #ADD dhparams.pem /etc/ssl/private/dhparams.pem

config/settings/common.py

@@ -14,7 +14,6 @@ import os
 import time
 import environ
 
-
 # from spirit.settings import *
 ROOT_DIR = environ.Path(__file__) - 3  # (mhackspace/config/settings/common.py - 3 = mhackspace/)
 APPS_DIR = ROOT_DIR.path('mhackspace')
@@ -341,6 +340,17 @@ LOGIN_URL = 'account_login'
 
 # SLUGLIFIER
 AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
+
+########## CELERY
+CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://')
+if CELERY_BROKER_URL == 'django://':
+    CELERY_RESULT_BACKEND = 'redis://'
+else:
+    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
+INSTALLED_APPS += ('django_celery_results','django_celery_beat',)
+########## END CELERY
+
 # django-compressor
 # ------------------------------------------------------------------------------
 INSTALLED_APPS += ("compressor", 'sass_processor',)

config/settings/local.py

@@ -83,6 +83,11 @@ INSTALLED_APPS += ('django_extensions', )
 # ------------------------------------------------------------------------------
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
+########## CELERY
+# In development, all tasks will be executed locally by blocking until the task returns
+CELERY_ALWAYS_EAGER = True
+########## END CELERY
+
 # Your local stuff: Below this line define 3rd party library settings
 # ------------------------------------------------------------------------------
 CAPTCHA = {

config/settings/production.py

@@ -184,7 +184,7 @@ LOGGING = {
     },
     'handlers': {
         'mail_admins': {
-            'level': 'ERROR',
+            'level': 'DEBUG',
             'filters': ['require_debug_false'],
             'class': 'django.utils.log.AdminEmailHandler'
         },

config/settings/stage.py

@@ -185,7 +185,7 @@ LOGGING = {
     },
     'handlers': {
         'mail_admins': {
-            'level': 'ERROR',
+            'level': 'DEBUG',
             'filters': ['require_debug_false'],
             'class': 'django.utils.log.AdminEmailHandler'
         },

dev.yml

@@ -10,8 +10,7 @@ services:
     volumes:
       - postgres_data_dev:/var/lib/postgresql/data
       - postgres_backup_dev:/backups
-    environment:
+    env_file: .env
-      - POSTGRES_USER=mhackspace
 
   django:
     build:
@@ -21,9 +20,6 @@ services:
     depends_on:
       - postgres
     env_file: .env
-    # environment:
-    #   - POSTGRES_USER=mhackspace
-    #   - USE_DOCKER=yes
     volumes:
       - .:/app
     ports:
@@ -42,5 +38,30 @@ services:
     command: npm install
     volumes:
       - ./:/usr/src/app
-  # redis:
+  redis:
-  #   image: redis:latest
+    image: redis:latest
+
+  celeryworker:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp worker -l INFO
+
+  celerybeat:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp beat -l debug 
+

docker-compose.yml

@@ -75,6 +75,6 @@ services:
 #      - TERM=xterm
 
 
-  # redis:
+  redis:
-  #   image: redis:latest
+    image: redis:latest
 

env.example

@@ -1,11 +1,16 @@
+#rename me to .env, and change per environment, .env should not be commited contains
+#sensitive settings
+
+#for dev only
+USE_DOCKER_DEBUG=yes
 
 # PostgreSQL
 POSTGRES_PASSWORD=mysecretpass
-POSTGRES_USER=postgresuser
+POSTGRES_USER=mhackspace
 
 # General settings
-DJANGO_ADMIN_URL=
+DJANGO_ADMIN_URL=admin
-DJANGO_SETTINGS_MODULE=config.settings.production
+DJANGO_SETTINGS_MODULE=config.settings.local
 DJANGO_SECRET_KEY=iud%k99yw!e+z+c12uatugbn=&lsdyd(t_byk9)@dp@lj6*c*n
 DJANGO_ALLOWED_HOSTS=.maidstone-hackspace.org.uk
 

live.yml

@@ -37,6 +37,29 @@ services:
     volumes:
       - ./:/usr/src/app
 
-  # redis:
+  redis:
-  #   image: redis:latest
+    image: redis:latest
 
+  celeryworker:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp worker -l INFO
+
+  celerybeat:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp beat -l INFO

mhackspace/base/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+
+from mhackspace.celeryapp import app as celery_app
+
+__all__ = ['celery_app']

mhackspace/base/tasks.py

@@ -0,0 +1,16 @@
+from celery import shared_task
+
+
+@shared_task
+def update_homepage_feeds():
+    pass
+    # import_feeds()
+
+# @task(bind=True)
+# @app.on_after_configure.connect
+# def setup_periodic_tasks(sender, **kwargs):
+#     # Executes every Monday morning at 7:30 a.m.
+#     sender.add_periodic_task(
+#         crontab(hour=4, minute=0),
+#         update_homepage_feeds.s(),
+#     )

mhackspace/celeryapp.py

@@ -0,0 +1,22 @@
+from __future__ import absolute_import
+import os
+from celery import Celery
+from celery.schedules import crontab
+# from django.apps import apps, AppConfig
+
+# if not settings.configured:
+    # set the default Django settings module for the 'celery' program.
+    # ;os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')  # pragma: no cover
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
+
+from django.conf import settings
+app = Celery('mhackspace')
+# app.config_from_object(settings)
+# app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks(lambda: ['mhackspace.base'])
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))  # pragma: no cover

mhackspace/subscriptions/tests/test_payment_gateways.py

@@ -43,7 +43,6 @@ class TestPaymentGatewaysGocardless(TestCase):
         self.assertEqual(result.get('reference'), '01')
         self.assertEqual(result.get('success'), 'success')
 
-    # @patch('mhackspace.subscriptions.payments.gocardless.request.requests.get', autospec=True)
     @patch('mhackspace.subscriptions.payments.gocardless.client.subscription', autospec=True)
     @patch('mhackspace.subscriptions.payments.gocardless.client.confirm_resource', autospec=True)
     def test_confirm_subscription_callback(self, mock_confirm, mock_subscription):

mhackspace/users/migrations/0004_auto_20170813_1557.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-08-13 15:57
+from __future__ import unicode_literals
+
+from django.db import migrations
+import stdimage.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0003_merge_20170226_0844'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='user',
+            name='image',
+        ),
+        migrations.AddField(
+            model_name='user',
+            name='_image',
+            field=stdimage.models.StdImageField(blank=True, db_column='image', null=True, upload_to='avatars/'),
+        ),
+    ]

requirements/base.txt

@@ -67,5 +67,9 @@ draceditor==1.1.8
 django-djconfig
 django-haystack
 git+https://github.com/nitely/Spirit.git
-# git+https://github.com/olymk2/django-xforwardedfor-middleware.git
 django-xforwardedfor-middleware==2.0
+
+# Application queue celery
+celery==4.1
+django-celery-results
+django-celery-beat

stage.yml

@@ -37,6 +37,29 @@ services:
     volumes:
       - ./:/usr/src/app
 
-  # redis:
+  redis:
-  #   image: redis:latest
+    image: redis:latest
 
+  celeryworker:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp worker -l INFO
+
+  celerybeat:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+      - redis
+    command: celery -A mhackspace.celeryapp beat -l INFO