diff --git a/config/settings/common.py b/config/settings/common.py index 3a7ad32..26e90c9 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -414,7 +414,11 @@ INSTALLED_APPS += ('django_extensions', ) INSTALLED_APPS += ('storages', ) INSTALLED_APPS += ('gunicorn', ) STATICFILES_FINDERS += ("compressor.finders.CompressorFinder", ) -STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +#STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +DEFAULT_FILE_STORAGE = 'mhackspace.core.storage.MediaStorage' +STATICFILES_STORAGE = 'mhackspace.core.storage.StaticStorage' +#COMPRESS_STORAGE = STATICFILES_STORAGE + # Location of root django.contrib.admin URL, use {% url 'admin:index' %} ADMIN_URL = '^trustee/' @@ -516,8 +520,10 @@ TWITTER_CONSUMER_SECRET=env('TWITTER_CONSUMER_SECRET') TWITTER_ACCESS_TOKEN=env('TWITTER_ACCESS_TOKEN') TWITTER_ACCESS_SECRET=env('TWITTER_ACCESS_SECRET') - -AWS_DEFAULT_ACL = None +LOCATION_PREFIX = env('BUCKET_PREFIX_PATH', default='') +MEDIAFILE_LOCATION = LOCATION_PREFIX + 'media' +STATICFILE_LOCATION = LOCATION_PREFIX + 'static' +AWS_DEFAULT_ACL = 'public-read' AWS_S3_OBJECT_PARAMETERS = { 'CacheControl': 'max-age=86400', } diff --git a/config/settings/local.py b/config/settings/local.py index 2cc2a08..f0f4799 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -108,7 +108,7 @@ TEMPLATE_DEBUG = True AWS_S3_SECURE_URLS = False AWS_ACCESS_KEY_ID = env('MINIO_ACCESS_KEY') AWS_SECRET_ACCESS_KEY = env('MINIO_SECRET_KEY') -AWS_STORAGE_BUCKET_NAME = 'static' +AWS_STORAGE_BUCKET_NAME = 'mhackspace-local' AWS_S3_ENDPOINT_URL = 'http://%s:9000' % socket.gethostbyname('bucket') AWS_S3_OBJECT_PARAMETERS = { 'CacheControl': 'max-age=86400', @@ -118,12 +118,11 @@ AWS_S3_SECURE_URLS = True STATIC_URL = '%s/%s/' % (AWS_S3_ENDPOINT_URL, AWS_STORAGE_BUCKET_NAME) -#STATICFILES_STORAGE = 'mhackspace.core.storage.SassStorageFix' # COMPRESSOR # ------------------------------------------------------------------------------ -COMPRESS_ENABLED = env.bool('COMPRESS_ENABLED', default=True) -COMPRESS_STORAGE = STATICFILES_STORAGE +#COMPRESS_ENABLED = env.bool('COMPRESS_ENABLED', default=True) +#COMPRESS_STORAGE = STATICFILES_STORAGE DEBUG_TOOLBAR_CONFIG = { 'INTERCEPT_REDIRECTS': False, } diff --git a/config/settings/production.py b/config/settings/production.py index 7aa17e1..56c1f47 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -194,7 +194,6 @@ AWS_S3_OBJECT_PARAMETERS = { AWS_LOCATION = 'static' STATIC_URL = '%s/%s/%s/' % (AWS_S3_ENDPOINT_URL, AWS_STORAGE_BUCKET_NAME, AWS_LOCATION) -STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' # COMPRESSOR # ------------------------------------------------------------------------------ diff --git a/config/settings/stage.py b/config/settings/stage.py index edcba8e..3ceb0c6 100644 --- a/config/settings/stage.py +++ b/config/settings/stage.py @@ -169,7 +169,6 @@ AWS_S3_OBJECT_PARAMETERS = { AWS_LOCATION = 'stage' STATIC_URL = '%s/%s/%s/' % (AWS_S3_ENDPOINT_URL, AWS_STORAGE_BUCKET_NAME, AWS_LOCATION) -STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' # COMPRESSOR # ------------------------------------------------------------------------------ diff --git a/local.yml b/local.yml index 4c66c90..00d8c5b 100644 --- a/local.yml +++ b/local.yml @@ -103,7 +103,7 @@ services: image: minio/minio env_file: .env volumes: - - ./mhackspace:/data + - ./bucket:/data command: server --config-dir /tmp/minio /data directory: diff --git a/mhackspace/blog/reader.py b/mhackspace/blog/reader.py new file mode 100644 index 0000000..2b5d876 --- /dev/null +++ b/mhackspace/blog/reader.py @@ -0,0 +1,16 @@ +import feedparser + +urls = [ + 'https://feeds.feedburner.com/projects-jl', + 'https://hackaday.com/tag/emf-camp-2018/feed/', + 'https://maidstone-hackspace.org.uk/blog/rss/', + 'http://webboggles.com/feed/', + 'https://blog.digitaloctave.com/rss.xml', +] + +for url in urls: + print(url) + parsed = feedparser.parse(url) + + for post in parsed.entries: + print(post.title) diff --git a/mhackspace/core/storage.py b/mhackspace/core/storage.py index 4563e25..e6ba4d5 100644 --- a/mhackspace/core/storage.py +++ b/mhackspace/core/storage.py @@ -4,3 +4,13 @@ from django.conf import settings class SassStorageFix(S3Boto3Storage): base_url = settings.AWS_S3_ENDPOINT_URL + + +class MediaStorage(S3Boto3Storage): + base_url = settings.AWS_S3_ENDPOINT_URL + location = settings.MEDIAFILE_LOCATION + + +class StaticStorage(S3Boto3Storage): + base_url = settings.AWS_S3_ENDPOINT_URL + location = settings.STATICFILE_LOCATION diff --git a/mhackspace/feeds/helper.py b/mhackspace/feeds/helper.py index 85a0240..0aa4176 100644 --- a/mhackspace/feeds/helper.py +++ b/mhackspace/feeds/helper.py @@ -1,33 +1,43 @@ # -*- coding: utf-8 -*- import os import logging +import feedparser from urllib.request import urlretrieve from django.core.files import File from django.utils.timezone import make_aware -from django.core.management import call_command from stdimage.utils import render_variations -from scaffold.readers.rss_reader import feed_reader + +# from scaffold.readers.rss_reader import feed_reader from mhackspace.feeds.models import Feed, Article, image_variations logger = logging.getLogger(__name__) +def feed_reader(feeds): + for feed in feeds: + print(feed) + yield feedparser.parse(feed['url']) + + def import_feeds(feed=False): remove_old_articles() rss_articles = feed_reader(get_active_feeds(feed)) articles = [] for article in rss_articles: - articles.append(Article( - url=article['url'].decode(), - feed=Feed.objects.get(pk=article['id']), - title=article['title'].decode(), - original_image=article['image'], - description=article['description'].decode(), - date=make_aware(article['date']) - )) + print(article) + articles.append( + Article( + url=article["url"].decode(), + feed=Feed.objects.get(pk=article["id"]), + title=article["title"].decode(), + original_image=article["image"], + description=article["description"].decode(), + date=make_aware(article["date"]), + ) + ) articles = Article.objects.bulk_create(articles) download_remote_images() @@ -48,14 +58,17 @@ def download_remote_images(): result = urlretrieve(article.original_image.__str__()) article.image.save( os.path.basename(article.original_image.__str__()), - File(open(result[0], 'rb')) + File(open(result[0], "rb")), ) render_variations(result[0], image_variations, replace=True) article.save() - except: + except Exception as e: logger.exception(result) logger.exception(result[0]) - logger.exception('Unable to download remote image for %s' % article.original_image) + logger.exception( + "Unable to download remote image for %s" + % article.original_image + ) def get_active_feeds(feed=False): @@ -68,9 +81,7 @@ def get_active_feeds(feed=False): for feed in feeds: if feed.enabled is False: continue - rss_feeds.append({ - 'id': feed.id, - 'author': feed.author, - 'url': feed.feed_url - }) + rss_feeds.append( + {"id": feed.id, "author": feed.author, "url": feed.feed_url} + ) return rss_feeds diff --git a/requirements/base.txt b/requirements/base.txt index 333638e..478642b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -104,3 +104,4 @@ python-magic==0.4.15 ldap3==2.5.1 bcrypt==3.1.4 python-twitter==3.4.2 +feedparser