Made Articles get stored in the DB and added image fetching

.gitignore

@@ -1,4 +1,5 @@
-media/*
+mhackspace/media/*
 .idea/
 __pycache__/
 node_modules
+src

config/settings/common.py

@@ -255,7 +255,7 @@ INSTALLED_APPS += ("compressor", )
 STATICFILES_FINDERS += ("compressor.finders.CompressorFinder", )
 
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
-ADMIN_URL = r'^admin/'
+ADMIN_URL = '^admin/'
 
 # Your common stuff: Below this line define 3rd party library settings
 # ------------------------------------------------------------------------------

config/urls.py

@@ -22,7 +22,7 @@ urlpatterns = [
     url(r'^members/$', MemberListView.as_view(), name='members'),
 
     # Django Admin, use {% url 'admin:index' %}
-    url(settings.ADMIN_URL, admin.site.urls),
+    url(r'{}'.format(settings.ADMIN_URL), admin.site.urls),
 
     # User management
     url(r'^users/', include('mhackspace.users.urls', namespace='users')),

mhackspace/feeds/admin.py

@@ -1,12 +1,38 @@
 from django.contrib import admin
 from django.contrib.admin import ModelAdmin
-from mhackspace.feeds.models import Feed
+from django.conf.urls import url
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+
+from mhackspace.feeds.models import Feed, Article
+from mhackspace.feeds.helper import import_feeds
 
 
 @admin.register(Feed)
 class FeedAdmin(ModelAdmin):
-    list_display = ('url', 'author', 'tags', 'enabled')
-    list_filter = ('enabled', )
+    list_display = ('title', 'home_url', 'author', 'tags', 'enabled')
+    list_filter = ('enabled',)
+
+
+@admin.register(Article)
+class ArticleAdmin(ModelAdmin):
+    date_hierarchy = 'date'
+    list_display = ('title', 'url', 'feed', 'date', 'displayed')
+    list_filter = ('displayed', 'feed', 'feed__author')
+    readonly_fields = ('original_image',)
+    ordering = ('-date',)
+
+    def get_urls(self):
+        urls = super(ArticleAdmin, self).get_urls()
+        my_urls = [
+            url(r'^import/$', self.admin_site.admin_view(self.import_articles))
+        ]
+        return my_urls + urls
+
+    def import_articles(self, request):
+        imported = import_feeds()
+        self.message_user(request, 'Successfully imported %s articles' % len(imported))
+        return HttpResponseRedirect(reverse('admin:feeds_article_changelist'))
 
 admin.site.site_title = 'Maidstone Hackspace Admin Area'
 admin.site.site_header = 'Maidstone Hackspace Admin Area'

mhackspace/feeds/fixtures/defaults.json

@@ -3,8 +3,10 @@
     "model": "feeds.feed",
     "pk": 1,
     "fields": {
-      "url": "http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss",
-      "author": "Simon Ridley",
+      "home_url": "http://thearduinoguy.org/",
+      "feed_url": "http://thearduinoguy.org/?feed=rss2",
+      "title": "The Arduino Guy",
+      "author": "Mike McRoberts",
       "tags": "",
       "image": "",
       "enabled": true
@@ -14,19 +16,23 @@
     "model": "feeds.feed",
     "pk": 2,
     "fields": {
-      "url": "http://www.matthewbeddow.co.uk/?feed=rss2",
-      "author": "Mathew Beddow",
-      "tags": "tech",
+      "home_url": "http://waistcoatforensicator.blogspot.com/",
+      "feed_url": "http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss",
+      "title": "Waistcoat Forensicator",
+      "author": "Simon Ridley",
+      "tags": "",
       "image": "",
-      "enabled": true
+      "enabled": false
     }
   },
   {
     "model": "feeds.feed",
     "pk": 3,
     "fields": {
-      "url": "http://blog.digitaloctave.co.uk/rss.xml",
-      "author": "Oliver Marks",
+      "home_url": "http://www.matthewbeddow.co.uk/",
+      "feed_url": "http://www.matthewbeddow.co.uk/?feed=rss2",
+      "title": "Mathew Beddow",
+      "author": "Mathew Beddow",
       "tags": "",
       "image": "",
       "enabled": true
@@ -36,8 +42,10 @@
     "model": "feeds.feed",
     "pk": 4,
     "fields": {
-      "url": "http://webboggles.com/feed/",
-      "author": "Ilya Titov",
+      "home_url": "http://blog.digitaloctave.co.uk",
+      "feed_url": "http://blog.digitaloctave.co.uk/rss.xml",
+      "title": "Oliver Marks",
+      "author": "Oliver Marks",
       "tags": "",
       "image": "",
       "enabled": true
@@ -47,8 +55,10 @@
     "model": "feeds.feed",
     "pk": 5,
     "fields": {
-      "url": "http://thearduinoguy.org/?feed=rss2",
-      "author": "Mike McRoberts",
+      "home_url": "http://webboggles.com/",
+      "feed_url": "http://webboggles.com/feed/",
+      "title": "Ilya Titov",
+      "author": "Ilya Titov",
       "tags": "",
       "image": "",
       "enabled": true
@@ -58,11 +68,97 @@
     "model": "feeds.feed",
     "pk": 6,
     "fields": {
-      "url": "https://feeds.feedburner.com/projects-jl",
-      "author": "James",
+      "home_url": "https://jamesleighton.com/",
+      "feed_url": "https://feeds.feedburner.com/projects-jl",
+      "title": "James Leighton",
+      "author": "James Leighton",
       "tags": "",
       "image": "",
       "enabled": false
     }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 507,
+    "fields": {
+      "url": "http://thearduinoguy.org/433mhz-radio-controlled-heating-controller-with-blynk-ui/",
+      "feed": 1,
+      "title": "433MHz Radio Controlled Heating Controller with Blynk UI",
+      "original_image": "http://thearduinoguy.org/wp-content/uploads/2016/12/Thermostat_bb-1-578x600.png",
+      "image": "",
+      "description": "<p>433MHz Radio Controlled Heating Controller with Blynk\u00a0UI This is a project to control your boiler using a 433MHz radio module. I have used a simple and cheap OOK radio module [\u2026]</p>",
+      "displayed": true,
+      "date": "2016-12-09T14:08:20Z"
+    }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 508,
+    "fields": {
+      "url": "http://webboggles.com/wacom-intuos-pro-manufacturing-fault-leads-to-loss-of-touch/",
+      "feed": 1,
+      "title": "Wacom Intuos Pro manufacturing fault leads to loss of touch.",
+      "original_image": "http://webboggles.com/wp-content/uploads/2016/12/IMG_20161201_215435.jpg",
+      "image": "",
+      "description": "<p>I love using my Wacom for work and carry it around on daily basis. Recently the touch stopped working. As the tablet is 3 years old and out of warranty I decided to disassemble it and peek inside to see if I could mend it myself. And here is the problem \u2014 the USB connector\u2019s \u2026 <a href=\"http://webboggles.com/wacom-intuos-pro-manufacturing-fault-leads-to-loss-of-touch/\" class=\"more-link\">Continue reading <span class=\"screen-reader-text\">Wacom Intuos Pro manufacturing fault leads to loss of touch.</span></a></p>",
+      "displayed": true,
+      "date": "2016-12-01T23:53:08Z"
+    }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 509,
+    "fields": {
+      "url": "http://thearduinoguy.org/neopixel-ring-clock/",
+      "feed": 1,
+      "title": "NeoPixel Ring Clock",
+      "original_image": "http://thearduinoguy.org/wp-content/uploads/2016/11/20161129_085040-1.jpg",
+      "image": "article/neopixel-ring-clock.jpg",
+      "description": "<p>I recently bought a 24 LED NeoPixel Ring. These things are great and I would highly recommend you getting one as they are great fun and very easy to use. [\u2026]</p>",
+      "displayed": true,
+      "date": "2016-11-29T09:11:17Z"
+    }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 510,
+    "fields": {
+      "url": "http://webboggles.com/spectacular-vr-for-nexus-5/",
+      "feed": 1,
+      "title": "Spectacular VR for Nexus 5",
+      "original_image": "http://webboggles.com/wp-content/uploads/2016/03/IMG_0985.jpg",
+      "image": "",
+      "description": "<p>This is a quick and easy 3D print that turns a Nexus 5 phone into a VR device, inspired by google cardboard. DIY: STL files on Thingiverse 2x M3 bolts and nuts are required, or just use cable ties. 2x 28mm bi-convex lenses (The lenses can be ordered on ebay for a couple of pounds)</p>",
+      "displayed": true,
+      "date": "2016-03-01T16:59:12Z"
+    }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 511,
+    "fields": {
+      "url": "http://thearduinoguy.org/lasertag-project/",
+      "feed": 1,
+      "title": "LaserTag Project",
+      "original_image": "http://thearduinoguy.org/wp-content/uploads/2016/08/LASERTAG-800x532.jpg",
+      "image": "article/lasertag-project.jpg",
+      "description": "<p>Along with some other guys from Maidstone Hackspace we have started work on a LaserTag project. I seem to have made the most progress so far and the prototype below [\u2026]</p>",
+      "displayed": true,
+      "date": "2016-08-31T09:57:13Z"
+    }
+  },
+  {
+    "model": "feeds.article",
+    "pk": 512,
+    "fields": {
+      "url": "http://webboggles.com/attiny85-game-kit-assembly-instructions/",
+      "feed": 1,
+      "title": "Attiny Arcade keychain game kit",
+      "original_image": "http://webboggles.com/wp-content/uploads/2016/01/IMG_3424-300x246.jpg",
+      "image": "",
+      "description": "<p>There has been a lot of updates to the Attiny Arcade keychain game project most notably the production of kits. I will add all the relevant information to this post and make it sticky. Buy via webboggles shop or on ebay. If you want to get in touch about the kit please tweet @webboggles, otherwise \u2026 <a href=\"http://webboggles.com/attiny85-game-kit-assembly-instructions/\" class=\"more-link\">Continue reading <span class=\"screen-reader-text\">Attiny Arcade keychain game kit</span></a></p>",
+      "displayed": true,
+      "date": "2016-01-17T00:34:00Z"
+    }
   }
 ]

mhackspace/feeds/helper.py

@@ -1,18 +1,78 @@
 # -*- coding: utf-8 -*-
-from django import template
-from mhackspace.feeds.models import Feed
+import os
+import logging
+
+from urllib.request import urlretrieve
+from django.core.files import File
+from django.utils.timezone import make_aware
+from django.core.management import call_command
 from scaffold.readers.rss_reader import feed_reader
 
-register = template.Library()
+from mhackspace.feeds.models import Feed, Article
+
+logger = logging.getLogger(__name__)
+
+
+def import_feeds(feed=False):
+    remove_old_articles()
+    rss_articles = feed_reader(get_active_feeds(feed))
+
+    articles = []
+    for article in rss_articles:
+        articles.append(Article(
+            url=article['url'],  # @olymk2 why cant I do article.url here?
+            feed=Feed.objects.get(pk=1),  # fixme: Nice hack :)
+            title=article['title'],
+            original_image=article['image'],
+            description=article['description'],
+            date=make_aware(article['date'])
+        ))
+
+    articles = Article.objects.bulk_create(articles)
+    download_remote_images()
+    return articles
+
+
+def remove_old_articles():
+    for article in Article.objects.all():
+        article.image.delete(save=False)
+    Article.objects.all().delete()
+
+
+def download_remote_images():
+    for article in Article.objects.all():
+        if not article.original_image:
+            continue
+        try:
+            result = urlretrieve(article.original_image.__str__())
+            article.image.save(
+                os.path.basename(article.original_image.__str__()),
+                File(open(result[0], 'rb'))
+            )
+            article.save()
+        except:
+            logger.exception('Unable to download remote image for %s' % article.original_image)
+    render_images()
+
+
+
+def render_images():
+    # todo: extract logic and manually render images
+    call_command('rendervariations', 'feeds.Article.image', '--replace')
+
+
+def get_active_feeds(feed=False):
+    if feed is not False:
+        feeds = Feed.objects.filter(pk__in=feed)
+    else:
+        feeds = Feed.objects.all()
 
-@register.inclusion_tag('feed_tiles.html')
-def fetch_feeds():
-    for feed
     rss_feeds = []
-    for feed in Feed.objects.all():
+    for feed in feeds:
+        if feed.enabled is False:
+            continue
         rss_feeds.append({
-            'author':'Simon Ridley',
-            'url': 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'
+            'author': feed.author,
+            'url': feed.feed_url
         })
-
-    return feed_reader(rss_feeds)
+    return rss_feeds

mhackspace/feeds/management/commands/importfeeds.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from mhackspace.feeds.helper import import_feeds
+
+
+class Command(BaseCommand):
+    help = 'Imports the RSS feeds from active blogs'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'blog_id',
+            nargs='*',
+            type=int,
+            default=False,
+            help='Specify a blog to get feeds form'
+        )
+
+    def handle(self, *args, **options):
+        imported = import_feeds(options['blog_id'])
+        self.stdout.write(self.style.SUCCESS('Successfully imported %s articles' % len(imported)))

mhackspace/feeds/migrations/0005_storing_articles.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.4 on 2017-01-08 03:42
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import stdimage.models
+import stdimage.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('feeds', '0004_feed_enabled'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Article',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField()),
+                ('title', models.CharField(max_length=255)),
+                ('original_image', models.URLField(blank=True, max_length=255, null=True)),
+                ('image', stdimage.models.StdImageField(blank=True, null=True, upload_to=stdimage.utils.UploadToAutoSlugClassNameDir('title'))),
+                ('description', models.TextField()),
+                ('displayed', models.BooleanField(default=True)),
+                ('date', models.DateTimeField(default=django.utils.timezone.now)),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='feed',
+            name='url',
+        ),
+        migrations.AddField(
+            model_name='feed',
+            name='feed_url',
+            field=models.URLField(default='http://thearduinoguy.org/?feed=rss2', verbose_name='RSS Feed URL'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='feed',
+            name='home_url',
+            field=models.URLField(default='http://thearduinoguy.org/', verbose_name='Site Home Page'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='feed',
+            name='title',
+            field=models.CharField(default='The Arduino Guy', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='feed',
+            name='id',
+            field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='feed',
+            name='image',
+            field=stdimage.models.StdImageField(blank=True, null=True, upload_to=stdimage.utils.UploadToAutoSlugClassNameDir('title')),
+        ),
+        migrations.AddField(
+            model_name='article',
+            name='feed',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='feeds.Feed'),
+        ),
+    ]

mhackspace/feeds/models.py

@@ -2,19 +2,54 @@
 from __future__ import unicode_literals, absolute_import
 
 from django.db import models
+from django.utils import timezone
 from django.utils.encoding import python_2_unicode_compatible
-from django.utils.translation import ugettext_lazy as _
+from stdimage.models import StdImageField
+from stdimage.utils import UploadToAutoSlugClassNameDir
+
+from mhackspace.users.models import User
 
 
 @python_2_unicode_compatible
 class Feed(models.Model):
-    id = models.AutoField(primary_key=True)
-    url = models.CharField(max_length=255)
+    home_url = models.URLField(verbose_name="Site Home Page")
+    feed_url = models.URLField(verbose_name="RSS Feed URL")
+    title = models.CharField(max_length=255)
     author = models.CharField(max_length=255)
     tags = models.CharField(max_length=255, blank=True)
-    image = models.ImageField(blank=True)
+    image = StdImageField(
+        upload_to=UploadToAutoSlugClassNameDir(populate_from='title'),
+        blank=True,
+        null=True,
+        variations={
+            'home': {
+                "width": 530,
+                "height": 220,
+                "crop": True}})
     enabled = models.BooleanField(default=True)
 
     def __str__(self):
-        return self.url
+        return self.title
 
+
+class Article(models.Model):
+    url = models.URLField()
+    feed = models.ForeignKey(Feed)
+    title = models.CharField(max_length=255)
+    original_image = models.URLField(max_length=255, blank=True, null=True)
+    image = StdImageField(
+        upload_to=UploadToAutoSlugClassNameDir(populate_from='title'),
+        blank=True,
+        null=True,
+        variations={
+            'home': {
+                "width": 530,
+                "height": 220,
+                "crop": True}})
+
+    description = models.TextField()
+    displayed = models.BooleanField(default=True)
+    date = models.DateTimeField(default=timezone.now)
+
+    def __str__(self):
+        return self.title

mhackspace/feeds/templatetags/feed_views.py

@@ -1,20 +1,9 @@
 # -*- coding: utf-8 -*-
 from django import template
-from mhackspace.feeds.models import Feed
-from scaffold.readers.rss_reader import feed_reader
+from mhackspace.feeds.models import Article
 
 register = template.Library()
 
 @register.inclusion_tag('feeds/list.html')
 def show_feeds():
-    rss_feeds = []
-    for feed in Feed.objects.all():
-        if feed.enabled is False:
-            continue
-        rss_feeds.append({
-            'author': feed.author,
-            'url': feed.url
-        })
-
-    result = feed_reader(rss_feeds)
-    return {'articles': [item for item in result]}
+    return {'articles': Article.objects.filter(displayed=True).select_related('feed').order_by('-date')}

mhackspace/static/css/project.css

@@ -6707,5 +6707,25 @@ body {
   line-height: 60px;
   background-color: #f5f5f5; }
 
+@media (max-width: 991px) {
+  .card-columns {
+    -moz-column-count: 2;
+         column-count: 2; } }
+
+@media (max-width: 767px) {
+  .card-columns {
+    -moz-column-count: 1;
+         column-count: 1; } }
+
+@media (max-width: 767px) {
+  .card {
+    margin-bottom: 0.75rem; } }
+
+.card-title {
+  color: #292b2c; }
+
+#feeds img {
+  width: 100%; }
+
 [hidden][style="display: block;"] {
   display: block !important; }

mhackspace/static/sass/pages/_home.scss

@@ -0,0 +1,23 @@
+.card-columns {
+  @include media-breakpoint-down(md) {
+    column-count: 2;
+  }
+  @include media-breakpoint-down(sm) {
+    column-count: 1;
+
+  }
+}
+
+.card {
+  @include media-breakpoint-down(sm) {
+    margin-bottom: 0.75rem;
+  }
+}
+
+.card-title {
+  color: $body-color;
+}
+
+#feeds img {
+  width: 100%;
+}

mhackspace/static/sass/project.scss

@@ -6,6 +6,8 @@
 @import "components/header";
 @import "components/footer";
 
+@import "pages/home";
+
 ////////////////////////////////
 		//Django Toolbar//
 ////////////////////////////////

mhackspace/templates/admin/feeds/article/change_list.html

@@ -0,0 +1,7 @@
+{% extends "admin/change_list.html" %}
+{% block object-tools-items %}
+  <li>
+      <a href="import/" class="grp-state-focus addlink">Import Articles</a>
+  </li>
+  {{ block.super }}
+{% endblock %}

mhackspace/templates/feeds/list.html

@@ -1,10 +1,15 @@
+{% load static %}
+{% get_static_prefix as STATIC_PREFIX %}
 <div class="card-columns" id="feeds">
   {% for article in articles %}
-    <div class="card" style="width: 20rem;">
-      <img class="card-img-top img-fluid" src="{{ article.image }}" alt="{{ article.title }}">
+    <div class="card">
+      <a href="{{ article.url }}">
+        <img class="card-img-top img-fluid" alt="{{ article.title }}"
+             src="{% firstof article.image.home.url article.feed.image.home.url "static/images/card.png" %}">
+      </a>
       <div class="card-block">
-        <h4 class="card-title">{{ article.title }}</h4>
-        <p class="card-text">{{ article.author }}</p>
+        <a href="{{ article.url }}" class="card-link"><h4 class="card-title">{{ article.title }}</h4></a>
+        <p class="card-text">{{ article.feed.author }}</p>
         <p class="card-text">{{ article.description|striptags }}</p>
         <a href="{{ article.url }}" class="card-link">View Original</a>
       </div>

mhackspace/users/migrations/0005_auto_20170108_0050.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.4 on 2017-01-08 00:50
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import stdimage.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0004_auto_20170106_2030'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='image',
+            field=stdimage.models.StdImageField(blank=True, null=True, upload_to='avatars/'),
+        ),
+        migrations.AlterField(
+            model_name='userblurb',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='users', to=settings.AUTH_USER_MODEL),
+        ),
+    ]