From 10039e0b0eaa46e068ed2a4711fc9f8d09b9ca05 Mon Sep 17 00:00:00 2001 From: Oliver Marks Date: Tue, 22 Aug 2017 22:47:44 +0100 Subject: [PATCH] celery integration tasks now working, improved feed image variation rendering --- .gitignore | 2 + config/settings/common.py | 16 +++++--- config/urls.py | 4 ++ dev.yml | 8 ++-- env.example | 3 ++ mhackspace/base/__init__.py | 2 - mhackspace/base/models.py | 21 ++++++++++ mhackspace/base/tasks.py | 4 +- mhackspace/celeryapp.py | 25 +++++------ mhackspace/feeds/helper.py | 11 ++--- mhackspace/feeds/models.py | 19 ++++----- mhackspace/requests/forms.py | 17 ++++++++ .../requests/migrations/0001_initial.py | 30 ++++++++++++++ .../0002_userrequests_created_date.py | 21 ++++++++++ .../0003_userrequests_request_type.py | 21 ++++++++++ mhackspace/requests/migrations/__init__.py | 0 mhackspace/requests/models.py | 22 ++++++++++ mhackspace/requests/views.py | 41 +++++++++++++++++++ mhackspace/templates/pages/requests.html | 40 ++++++++++++++++++ mhackspace/users/__init__.py | 4 +- mhackspace/users/tasks.py | 8 ++++ requirements/base.txt | 9 ++-- 22 files changed, 278 insertions(+), 50 deletions(-) create mode 100644 mhackspace/requests/forms.py create mode 100644 mhackspace/requests/migrations/0001_initial.py create mode 100644 mhackspace/requests/migrations/0002_userrequests_created_date.py create mode 100644 mhackspace/requests/migrations/0003_userrequests_request_type.py create mode 100644 mhackspace/requests/migrations/__init__.py create mode 100644 mhackspace/requests/models.py create mode 100644 mhackspace/requests/views.py create mode 100644 mhackspace/templates/pages/requests.html create mode 100644 mhackspace/users/tasks.py diff --git a/.gitignore b/.gitignore index 7cf220f..9badde2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ src .env staticfiles/* cache/ +celerybeat-schedule + diff --git a/config/settings/common.py b/config/settings/common.py index f0661ab..980156d 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -124,6 +124,7 @@ LOCAL_APPS = ( 'mhackspace.contact', 'mhackspace.members', 'mhackspace.blog', + 'mhackspace.requests', ) # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps @@ -342,13 +343,18 @@ LOGIN_URL = 'account_login' AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify' ########## CELERY -CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://') -if CELERY_BROKER_URL == 'django://': - CELERY_RESULT_BACKEND = 'redis://' -else: - CELERY_RESULT_BACKEND = CELERY_BROKER_URL +# CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='redis://') +CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='redis://redis:6379/0') +#if CELERY_BROKER_URL == 'django://': +# CELERY_RESULT_BACKEND = 'redis://' +#else: +CELERY_RESULT_BACKEND = 'django-cache' CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' INSTALLED_APPS += ('django_celery_results','django_celery_beat',) +CELERY_TIMEZONE = 'UTC' +CELERY_ENABLE_UTC = True +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' ########## END CELERY # django-compressor diff --git a/config/urls.py b/config/urls.py index a16ed4f..56d7cba 100644 --- a/config/urls.py +++ b/config/urls.py @@ -19,6 +19,8 @@ from mhackspace.base.views import markdown_uploader from mhackspace.blog.views import PostViewSet, CategoryViewSet, BlogPost, PostList from mhackspace.blog.sitemaps import PostSitemap, CategorySitemap from mhackspace.feeds.views import FeedViewSet, ArticleViewSet +from mhackspace.requests.views import RequestsForm, RequestsList + # import spirit.urls router = DefaultRouter() @@ -38,6 +40,8 @@ urlpatterns = [ url(r'^chat/$', TemplateView.as_view(template_name='pages/chat.html'), name='chat'), url(r'^mailing-list/$', TemplateView.as_view(template_name='pages/mailing-list.html'), name='group'), url(r'^contact/$', contact, name='contact'), + url(r'^requests/$', RequestsList.as_view(), name='requests'), + url(r'^requests/create$', RequestsForm.as_view(), name='requests_form'), url(r'^discuss/', include('spirit.urls')), url(r'^api/v1/', include(router.urls, namespace='v1')), diff --git a/dev.yml b/dev.yml index 2ece4d4..898fc24 100644 --- a/dev.yml +++ b/dev.yml @@ -38,13 +38,14 @@ services: command: npm install volumes: - ./:/usr/src/app + redis: image: redis:latest celeryworker: build: context: . - dockerfile: ./compose/django/Dockerfile + dockerfile: ./compose/django/Dockerfile-dev env_file: .env volumes: - .:/app @@ -56,12 +57,13 @@ services: celerybeat: build: context: . - dockerfile: ./compose/django/Dockerfile + dockerfile: ./compose/django/Dockerfile-dev env_file: .env volumes: - .:/app depends_on: - postgres - redis - command: celery -A mhackspace.celeryapp beat -l debug + command: celery -A mhackspace.celeryapp beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler + #command: sleep 1h diff --git a/env.example b/env.example index 1697a8e..87a5eb1 100644 --- a/env.example +++ b/env.example @@ -7,6 +7,7 @@ USE_DOCKER_DEBUG=yes # PostgreSQL POSTGRES_PASSWORD=mysecretpass POSTGRES_USER=mhackspace +DATABASE_URL=postgres://mhackspace:mysecretpass@postgres:5432/mhackspace # General settings DJANGO_ADMIN_URL=admin @@ -33,6 +34,8 @@ DJANGO_ACCOUNT_ALLOW_REGISTRATION=True COMPRESS_ENABLED= +REDIS_URL=redis://redis:6379 +CELERY_BROKER_URL=redis://redis:6379/0 PAYMENT_ENVIRONMENT=sandbox PAYMENT_REDIRECT_URL=http://127.0.0.1:8180 diff --git a/mhackspace/base/__init__.py b/mhackspace/base/__init__.py index 8eb049c..70b88d3 100644 --- a/mhackspace/base/__init__.py +++ b/mhackspace/base/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - from mhackspace.celeryapp import app as celery_app __all__ = ['celery_app'] diff --git a/mhackspace/base/models.py b/mhackspace/base/models.py index 4002ced..adb810d 100644 --- a/mhackspace/base/models.py +++ b/mhackspace/base/models.py @@ -7,6 +7,10 @@ from stdimage.models import StdImageField from stdimage.utils import UploadToAutoSlugClassNameDir from stdimage.validators import MinSizeValidator +from spirit.comment.models import Comment +# from django.contrib.auth.models import User +from django.core.mail import EmailMessage +from django.db.models.signals import post_save class BannerImage(models.Model): url = models.URLField() @@ -47,3 +51,20 @@ class BannerImage(models.Model): def __str__(self): return self.title + + +# should be done inside spirit +def send_topic_update_email(sender, instance, **kwargs): + comments = Comment.objects.filter(topic=instance.topic) + addresses = {comment.user.email for comment in comments} + for user_email in addresses: + email = EmailMessage( + '[%s] - %s' % ('MH', instance.topic.title), + 'A topic you have interacted with has been updated click link to see new comments %s' % instance.get_absolute_url(), + 'no-reply@maidstone-hackspace.org.uk', + to=[user_email], + headers={'Reply-To': 'no-reply@maidstone-hackspace.org.uk'}) + email.send() + + +post_save.connect(send_topic_update_email, sender=Comment) diff --git a/mhackspace/base/tasks.py b/mhackspace/base/tasks.py index 8328b1f..f1718c8 100644 --- a/mhackspace/base/tasks.py +++ b/mhackspace/base/tasks.py @@ -1,10 +1,10 @@ from celery import shared_task +from mhackspace.feeds.helper import import_feeds @shared_task def update_homepage_feeds(): - pass - # import_feeds() + return import_feeds() # @task(bind=True) # @app.on_after_configure.connect diff --git a/mhackspace/celeryapp.py b/mhackspace/celeryapp.py index 5c09831..4315587 100644 --- a/mhackspace/celeryapp.py +++ b/mhackspace/celeryapp.py @@ -1,21 +1,18 @@ -from __future__ import absolute_import +# from __future__ import absolute_import import os -from celery import Celery -from celery.schedules import crontab -# from django.apps import apps, AppConfig - -# if not settings.configured: - # set the default Django settings module for the 'celery' program. - # ;os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') # pragma: no cover - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') - from django.conf import settings +from celery import Celery + + +if not settings.configured: + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') + app = Celery('mhackspace') -# app.config_from_object(settings) -# app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) app.config_from_object('django.conf:settings', namespace='CELERY') -app.autodiscover_tasks(lambda: ['mhackspace.base']) + +# app.autodiscover_tasks(lambda: ['mhackspace.base'], related_name='tasks') +app.autodiscover_tasks() + @app.task(bind=True) def debug_task(self): diff --git a/mhackspace/feeds/helper.py b/mhackspace/feeds/helper.py index 0518811..59a72af 100644 --- a/mhackspace/feeds/helper.py +++ b/mhackspace/feeds/helper.py @@ -6,9 +6,10 @@ from urllib.request import urlretrieve from django.core.files import File from django.utils.timezone import make_aware from django.core.management import call_command +from stdimage.utils import render_variations from scaffold.readers.rss_reader import feed_reader -from mhackspace.feeds.models import Feed, Article +from mhackspace.feeds.models import Feed, Article, image_variations logger = logging.getLogger(__name__) @@ -49,16 +50,10 @@ def download_remote_images(): os.path.basename(article.original_image.__str__()), File(open(result[0], 'rb')) ) + render_variations(result[0], image_variations, replace=True) article.save() except: logger.exception('Unable to download remote image for %s' % article.original_image) - render_images() - - - -def render_images(): - # todo: extract logic and manually render images - call_command('rendervariations', 'feeds.Article.image', '--replace') def get_active_feeds(feed=False): diff --git a/mhackspace/feeds/models.py b/mhackspace/feeds/models.py index 680329b..105b35c 100644 --- a/mhackspace/feeds/models.py +++ b/mhackspace/feeds/models.py @@ -7,7 +7,12 @@ from django.utils.encoding import python_2_unicode_compatible from stdimage.models import StdImageField from stdimage.utils import UploadToAutoSlugClassNameDir -from mhackspace.users.models import User + +image_variations = { + 'home': { + "width": 530, + "height": 220, + "crop": True}} @python_2_unicode_compatible @@ -21,11 +26,7 @@ class Feed(models.Model): upload_to=UploadToAutoSlugClassNameDir(populate_from='title'), blank=True, null=True, - variations={ - 'home': { - "width": 530, - "height": 220, - "crop": True}}) + variations=image_variations) enabled = models.BooleanField(default=True) def __str__(self): @@ -41,11 +42,7 @@ class Article(models.Model): upload_to=UploadToAutoSlugClassNameDir(populate_from='title'), blank=True, null=True, - variations={ - 'home': { - "width": 530, - "height": 220, - "crop": True}}) + variations=image_variations) description = models.TextField() displayed = models.BooleanField(default=True) diff --git a/mhackspace/requests/forms.py b/mhackspace/requests/forms.py new file mode 100644 index 0000000..9fb34b0 --- /dev/null +++ b/mhackspace/requests/forms.py @@ -0,0 +1,17 @@ +from django import forms +from mhackspace.requests.models import UserRequests +from mhackspace.requests.models import REQUEST_TYPES + + +class UserRequestForm(forms.ModelForm): + class Meta: + model = UserRequests + exclude = ['user', 'created_date'] + # description = forms.CharField( + # required=True, + # widget=forms.Textarea + # ) + # request_type = forms.ChoiceField( + # required=True, + # widget=forms.Select, + # choices=REQUEST_TYPES) diff --git a/mhackspace/requests/migrations/0001_initial.py b/mhackspace/requests/migrations/0001_initial.py new file mode 100644 index 0000000..50a7b91 --- /dev/null +++ b/mhackspace/requests/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-08-15 19:53 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='UserRequests', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField()), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ('pk',), + }, + ), + ] diff --git a/mhackspace/requests/migrations/0002_userrequests_created_date.py b/mhackspace/requests/migrations/0002_userrequests_created_date.py new file mode 100644 index 0000000..d5ff151 --- /dev/null +++ b/mhackspace/requests/migrations/0002_userrequests_created_date.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-08-15 21:36 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('requests', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='userrequests', + name='created_date', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/mhackspace/requests/migrations/0003_userrequests_request_type.py b/mhackspace/requests/migrations/0003_userrequests_request_type.py new file mode 100644 index 0000000..6407670 --- /dev/null +++ b/mhackspace/requests/migrations/0003_userrequests_request_type.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-08-15 21:39 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('requests', '0002_userrequests_created_date'), + ] + + operations = [ + migrations.AddField( + model_name='userrequests', + name='request_type', + field=models.IntegerField(choices=[(1, 'Equipment request'), (2, 'Educational request'), (3, 'Training request')], default=1), + preserve_default=False, + ), + ] diff --git a/mhackspace/requests/migrations/__init__.py b/mhackspace/requests/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mhackspace/requests/models.py b/mhackspace/requests/models.py new file mode 100644 index 0000000..ed388e4 --- /dev/null +++ b/mhackspace/requests/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +from django.conf import settings +from django.db import models +from django.utils import timezone + + +REQUEST_TYPES = ( + (1, 'Equipment request'), + (2, 'Educational request'), + (3, 'Training request')) + + +class UserRequests(models.Model): + user = models.OneToOneField( + settings.AUTH_USER_MODEL, related_name='+') + request_type = models.IntegerField(choices=REQUEST_TYPES) + description = models.TextField() + created_date = models.DateTimeField(default=timezone.now) + + class Meta: + ordering = ('pk',) diff --git a/mhackspace/requests/views.py b/mhackspace/requests/views.py new file mode 100644 index 0000000..db01af3 --- /dev/null +++ b/mhackspace/requests/views.py @@ -0,0 +1,41 @@ +from django.core.mail import EmailMessage +from django.contrib import messages +from mhackspace.requests.forms import UserRequestForm +from mhackspace.requests.models import UserRequests +from django.views.generic import ListView +from django.views.generic.edit import FormView + + +class RequestsForm(FormView): + template_name = 'pages/requests.html' + form_class = UserRequestForm + success_url = '/requests' + + def form_valid(self, form): + if form.is_valid(): + obj = form.save(commit=False) + obj.user = self.request.user + obj.save() + # email = EmailMessage( + # '[%s] - %s' % (data['enquiry_type'], data['subject']), + # data['message'], + # data['contact_email'], + # to=['contact@maidstone-hackspace.org.uk'], + # headers={'Reply-To': data['contact_email']}) + # email.send() + messages.add_message(self.request, messages.INFO, 'Request successfully made.') + + return super(FormView, self).form_valid(form) + + +class RequestsList(ListView): + template_name = 'pages/requests.html' + model = UserRequests + context_object_name = 'requests' + paginate_by = 5 + + # def get_queryset(self): + # if 'category' in self.kwargs: + # self.category = get_object_or_404(Category, slug=self.kwargs['category']) + # return Post.objects.filter(active=True, categories=self.category, published_date__lte=timezone.now(), members_only=False) + # return Post.objects.filter(active=True, published_date__lte=timezone.now(), members_only=False) diff --git a/mhackspace/templates/pages/requests.html b/mhackspace/templates/pages/requests.html new file mode 100644 index 0000000..f82e429 --- /dev/null +++ b/mhackspace/templates/pages/requests.html @@ -0,0 +1,40 @@ +{% extends "base.html" %} +{% load i18n %} +{% load crispy_forms_tags %} +{% load recapture %} + +{% block content %} +

Requests

+ + Make a request for equipment you would like to see in the space, request training on equipment or ak a member to run a workshop or talk on something. +
+ + {% if form %} +
+ {% csrf_token %} + {{ form|crispy }} + {{ google_capture }} + +
+ {% endif %} + + +{% for request in requests %} + + + + + +{% endfor %} +
+ {{ request.type }} + + {{ request.description }} +
+ +{% endblock content %} + +{% block javascript %} + {{ block.super }} + +{% endblock %} diff --git a/mhackspace/users/__init__.py b/mhackspace/users/__init__.py index 40a96af..70b88d3 100644 --- a/mhackspace/users/__init__.py +++ b/mhackspace/users/__init__.py @@ -1 +1,3 @@ -# -*- coding: utf-8 -*- +from mhackspace.celeryapp import app as celery_app + +__all__ = ['celery_app'] diff --git a/mhackspace/users/tasks.py b/mhackspace/users/tasks.py new file mode 100644 index 0000000..fa0802e --- /dev/null +++ b/mhackspace/users/tasks.py @@ -0,0 +1,8 @@ +# from celery import shared_task +# from mhackspace.subscriptions.management.commands.update_membership_status import update_subscriptions + + +# @shared_task +# def update_users_memebership_status(): +# for user in update_subscriptions(provider_name='gocardless'): +# continue diff --git a/requirements/base.txt b/requirements/base.txt index 9d79f8c..98b4bc4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,7 +5,7 @@ wheel==0.29.0 # Bleeding edge Django -django==1.11.3 +django==1.11.4 # Configuration django-environ==0.4.3 @@ -70,6 +70,7 @@ git+https://github.com/nitely/Spirit.git django-xforwardedfor-middleware==2.0 # Application queue celery -celery==4.1 -django-celery-results -django-celery-beat +celery==4.1.0 +django-celery-results==1.0.1 +django-celery-beat==1.0.1 +