celery integration tasks now working, improved feed image variation rendering

.gitignore

@@ -6,3 +6,5 @@ src
 .env
 staticfiles/*
 cache/
+celerybeat-schedule
+

config/settings/common.py

@@ -124,6 +124,7 @@ LOCAL_APPS = (
     'mhackspace.contact',
     'mhackspace.members',
     'mhackspace.blog',
+    'mhackspace.requests',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -342,13 +343,18 @@ LOGIN_URL = 'account_login'
 AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
 
 ########## CELERY
-CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://')
+# CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='redis://')
-if CELERY_BROKER_URL == 'django://':
+CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='redis://redis:6379/0')
-    CELERY_RESULT_BACKEND = 'redis://'
+#if CELERY_BROKER_URL == 'django://':
-else:
+# CELERY_RESULT_BACKEND = 'redis://'
-    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+#else:
+CELERY_RESULT_BACKEND = 'django-cache'
 CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
 INSTALLED_APPS += ('django_celery_results','django_celery_beat',)
+CELERY_TIMEZONE = 'UTC'
+CELERY_ENABLE_UTC = True
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
 ########## END CELERY
 
 # django-compressor

config/urls.py

@@ -19,6 +19,8 @@ from mhackspace.base.views import markdown_uploader
 from mhackspace.blog.views import PostViewSet, CategoryViewSet, BlogPost, PostList
 from mhackspace.blog.sitemaps import PostSitemap, CategorySitemap
 from mhackspace.feeds.views import FeedViewSet, ArticleViewSet
+from mhackspace.requests.views import RequestsForm, RequestsList
+
 
 # import spirit.urls
 router = DefaultRouter()
@@ -38,6 +40,8 @@ urlpatterns = [
     url(r'^chat/$', TemplateView.as_view(template_name='pages/chat.html'), name='chat'),
     url(r'^mailing-list/$', TemplateView.as_view(template_name='pages/mailing-list.html'), name='group'),
     url(r'^contact/$', contact, name='contact'),
+    url(r'^requests/$', RequestsList.as_view(), name='requests'),
+    url(r'^requests/create$', RequestsForm.as_view(), name='requests_form'),
 
     url(r'^discuss/', include('spirit.urls')),
     url(r'^api/v1/', include(router.urls, namespace='v1')),

dev.yml

@@ -38,13 +38,14 @@ services:
     command: npm install
     volumes:
       - ./:/usr/src/app
+
   redis:
     image: redis:latest
 
   celeryworker:
     build:
       context: .
-      dockerfile: ./compose/django/Dockerfile
+      dockerfile: ./compose/django/Dockerfile-dev
     env_file: .env
     volumes:
       - .:/app
@@ -56,12 +57,13 @@ services:
   celerybeat:
     build:
       context: .
-      dockerfile: ./compose/django/Dockerfile
+      dockerfile: ./compose/django/Dockerfile-dev
     env_file: .env
     volumes:
       - .:/app
     depends_on:
       - postgres
       - redis
-    command: celery -A mhackspace.celeryapp beat -l debug 
+    command: celery -A mhackspace.celeryapp beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    #command: sleep 1h 
 

env.example

@@ -7,6 +7,7 @@ USE_DOCKER_DEBUG=yes
 # PostgreSQL
 POSTGRES_PASSWORD=mysecretpass
 POSTGRES_USER=mhackspace
+DATABASE_URL=postgres://mhackspace:mysecretpass@postgres:5432/mhackspace
 
 # General settings
 DJANGO_ADMIN_URL=admin
@@ -33,6 +34,8 @@ DJANGO_ACCOUNT_ALLOW_REGISTRATION=True
 
 COMPRESS_ENABLED=
 
+REDIS_URL=redis://redis:6379
+CELERY_BROKER_URL=redis://redis:6379/0
 
 PAYMENT_ENVIRONMENT=sandbox
 PAYMENT_REDIRECT_URL=http://127.0.0.1:8180

mhackspace/base/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
 from mhackspace.celeryapp import app as celery_app
 
 __all__ = ['celery_app']

mhackspace/base/models.py

@@ -7,6 +7,10 @@ from stdimage.models import StdImageField
 from stdimage.utils import UploadToAutoSlugClassNameDir
 from stdimage.validators import MinSizeValidator
 
+from spirit.comment.models import Comment
+# from django.contrib.auth.models import User
+from django.core.mail import EmailMessage
+from django.db.models.signals import post_save
 
 class BannerImage(models.Model):
     url = models.URLField()
@@ -47,3 +51,20 @@ class BannerImage(models.Model):
 
     def __str__(self):
         return self.title
+
+
+# should be done inside spirit
+def send_topic_update_email(sender, instance, **kwargs):
+    comments = Comment.objects.filter(topic=instance.topic)
+    addresses = {comment.user.email for comment in comments}
+    for user_email in addresses:
+        email = EmailMessage(
+            '[%s] - %s' % ('MH', instance.topic.title),
+            'A topic you have interacted with has been updated click link to see new comments %s' % instance.get_absolute_url(),
+            'no-reply@maidstone-hackspace.org.uk',
+            to=[user_email],
+            headers={'Reply-To': 'no-reply@maidstone-hackspace.org.uk'})
+        email.send()
+
+
+post_save.connect(send_topic_update_email, sender=Comment)

mhackspace/base/tasks.py

@@ -1,10 +1,10 @@
 from celery import shared_task
+from mhackspace.feeds.helper import import_feeds
 
 
 @shared_task
 def update_homepage_feeds():
-    pass
+    return import_feeds()
-    # import_feeds()
 
 # @task(bind=True)
 # @app.on_after_configure.connect

mhackspace/celeryapp.py

@@ -1,21 +1,18 @@
-from __future__ import absolute_import
+# from __future__ import absolute_import
 import os
-from celery import Celery
-from celery.schedules import crontab
-# from django.apps import apps, AppConfig
-
-# if not settings.configured:
-    # set the default Django settings module for the 'celery' program.
-    # ;os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')  # pragma: no cover
-
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
-
 from django.conf import settings
+from celery import Celery
+
+
+if not settings.configured:
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
+
 app = Celery('mhackspace')
-# app.config_from_object(settings)
-# app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 app.config_from_object('django.conf:settings', namespace='CELERY')
-app.autodiscover_tasks(lambda: ['mhackspace.base'])
+
+# app.autodiscover_tasks(lambda: ['mhackspace.base'], related_name='tasks')
+app.autodiscover_tasks()
+
 
 @app.task(bind=True)
 def debug_task(self):

mhackspace/feeds/helper.py

@@ -6,9 +6,10 @@ from urllib.request import urlretrieve
 from django.core.files import File
 from django.utils.timezone import make_aware
 from django.core.management import call_command
+from stdimage.utils import render_variations
 from scaffold.readers.rss_reader import feed_reader
 
-from mhackspace.feeds.models import Feed, Article
+from mhackspace.feeds.models import Feed, Article, image_variations
 
 logger = logging.getLogger(__name__)
 
@@ -49,16 +50,10 @@ def download_remote_images():
                 os.path.basename(article.original_image.__str__()),
                 File(open(result[0], 'rb'))
             )
+            render_variations(result[0], image_variations, replace=True)
             article.save()
         except:
             logger.exception('Unable to download remote image for %s' % article.original_image)
-    render_images()
-
-
-
-def render_images():
-    # todo: extract logic and manually render images
-    call_command('rendervariations', 'feeds.Article.image', '--replace')
 
 
 def get_active_feeds(feed=False):

mhackspace/feeds/models.py

@@ -7,7 +7,12 @@ from django.utils.encoding import python_2_unicode_compatible
 from stdimage.models import StdImageField
 from stdimage.utils import UploadToAutoSlugClassNameDir
 
-from mhackspace.users.models import User
+
+image_variations = {
+    'home': {
+        "width": 530,
+        "height": 220,
+        "crop": True}}
 
 
 @python_2_unicode_compatible
@@ -21,11 +26,7 @@ class Feed(models.Model):
         upload_to=UploadToAutoSlugClassNameDir(populate_from='title'),
         blank=True,
         null=True,
-        variations={
+        variations=image_variations)
-            'home': {
-                "width": 530,
-                "height": 220,
-                "crop": True}})
     enabled = models.BooleanField(default=True)
 
     def __str__(self):
@@ -41,11 +42,7 @@ class Article(models.Model):
         upload_to=UploadToAutoSlugClassNameDir(populate_from='title'),
         blank=True,
         null=True,
-        variations={
+        variations=image_variations)
-            'home': {
-                "width": 530,
-                "height": 220,
-                "crop": True}})
 
     description = models.TextField()
     displayed = models.BooleanField(default=True)

mhackspace/requests/forms.py

@@ -0,0 +1,17 @@
+from django import forms
+from mhackspace.requests.models import UserRequests
+from mhackspace.requests.models import REQUEST_TYPES
+
+
+class UserRequestForm(forms.ModelForm):
+    class Meta:
+        model = UserRequests
+        exclude = ['user', 'created_date']
+    # description = forms.CharField(
+    #     required=True,
+    #     widget=forms.Textarea
+    # )
+    # request_type = forms.ChoiceField(
+    #     required=True,
+    #     widget=forms.Select,
+    #     choices=REQUEST_TYPES)

mhackspace/requests/migrations/0001_initial.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-08-15 19:53
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UserRequests',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('description', models.TextField()),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ('pk',),
+            },
+        ),
+    ]

mhackspace/requests/migrations/0002_userrequests_created_date.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-08-15 21:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('requests', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userrequests',
+            name='created_date',
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+    ]

mhackspace/requests/migrations/0003_userrequests_request_type.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-08-15 21:39
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('requests', '0002_userrequests_created_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userrequests',
+            name='request_type',
+            field=models.IntegerField(choices=[(1, 'Equipment request'), (2, 'Educational request'), (3, 'Training request')], default=1),
+            preserve_default=False,
+        ),
+    ]

mhackspace/requests/models.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+
+REQUEST_TYPES = (
+    (1, 'Equipment request'),
+    (2, 'Educational request'),
+    (3, 'Training request'))
+
+
+class UserRequests(models.Model):
+    user = models.OneToOneField(
+        settings.AUTH_USER_MODEL, related_name='+')
+    request_type = models.IntegerField(choices=REQUEST_TYPES)
+    description = models.TextField()
+    created_date = models.DateTimeField(default=timezone.now)
+
+    class Meta:
+        ordering = ('pk',)

mhackspace/requests/views.py

@@ -0,0 +1,41 @@
+from django.core.mail import EmailMessage
+from django.contrib import messages
+from mhackspace.requests.forms import UserRequestForm
+from mhackspace.requests.models import UserRequests
+from django.views.generic import ListView
+from django.views.generic.edit import FormView
+
+
+class RequestsForm(FormView):
+    template_name = 'pages/requests.html'
+    form_class = UserRequestForm
+    success_url = '/requests'
+
+    def form_valid(self, form):
+        if form.is_valid():
+            obj = form.save(commit=False)
+            obj.user = self.request.user
+            obj.save()
+            # email = EmailMessage(
+            #     '[%s] - %s' % (data['enquiry_type'], data['subject']),
+            #     data['message'],
+            #     data['contact_email'],
+            #     to=['contact@maidstone-hackspace.org.uk'],
+            #     headers={'Reply-To': data['contact_email']})
+            # email.send()
+            messages.add_message(self.request, messages.INFO, 'Request successfully made.')
+
+        return super(FormView, self).form_valid(form)
+
+
+class RequestsList(ListView):
+    template_name = 'pages/requests.html'
+    model = UserRequests
+    context_object_name = 'requests'
+    paginate_by = 5
+
+    # def get_queryset(self):
+    #     if 'category' in self.kwargs:
+    #         self.category = get_object_or_404(Category, slug=self.kwargs['category'])
+    #         return Post.objects.filter(active=True, categories=self.category, published_date__lte=timezone.now(), members_only=False)
+    #     return Post.objects.filter(active=True, published_date__lte=timezone.now(), members_only=False)

mhackspace/templates/pages/requests.html

@@ -0,0 +1,40 @@
+{% extends "base.html" %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+{% load recapture %}
+
+{% block content %}
+  <h2>Requests</h2>
+
+  Make a request for equipment you would like to see in the space, request training on equipment or ak a member to run a workshop or talk on something.
+  <br />
+
+  {% if form %}
+    <form method="POST" action="{% url 'requests_form' %}" class="requests_form">
+      {% csrf_token %}
+      {{ form|crispy }}
+      {{ google_capture }}
+      <button class="btn btn-primary" type="submit" name="action">{% trans "Send" %}</button>
+    </form>
+  {% endif %}
+
+    <table>
+{% for request in requests %}
+      <tr>
+        <td>
+          {{ request.type }}
+        </td>
+        <td>
+          {{ request.description }}
+        </td>
+      </tr>
+
+{% endfor %}
+    </table>
+
+{% endblock content %}
+
+{% block javascript %}
+  {{ block.super }}
+  <script src='https://www.google.com/recaptcha/api.js'></script>
+{% endblock %}

mhackspace/users/__init__.py

@@ -1 +1,3 @@
-# -*- coding: utf-8 -*-
+from mhackspace.celeryapp import app as celery_app
+
+__all__ = ['celery_app']

mhackspace/users/tasks.py

@@ -0,0 +1,8 @@
+# from celery import shared_task
+# from mhackspace.subscriptions.management.commands.update_membership_status import update_subscriptions
+
+
+# @shared_task
+# def update_users_memebership_status():
+#     for user in update_subscriptions(provider_name='gocardless'):
+#         continue

requirements/base.txt

@@ -5,7 +5,7 @@
 wheel==0.29.0
 
 # Bleeding edge Django
-django==1.11.3
+django==1.11.4
 
 # Configuration
 django-environ==0.4.3
@@ -70,6 +70,7 @@ git+https://github.com/nitely/Spirit.git
 django-xforwardedfor-middleware==2.0
 
 # Application queue celery
-celery==4.1
+celery==4.1.0
-django-celery-results
+django-celery-results==1.0.1
-django-celery-beat
+django-celery-beat==1.0.1
+