rss agreggator code

site/constants.py

@@ -5,6 +5,7 @@ page_menu = [
     ('Contact', '#mailing-list-signup')]
 
 banner_images = [
+    ('/static/template/images/hackspace-banner.png', 'http://maidstone-hackspace.org.uk/', 'title', 'intro text'),
     ('/static/template/images/example-01.jpg', 'http://www.google.co.uk', 'title', 'intro text'),
     ('/static/template/images/example-02.jpg', 'http://www.google.co.uk', 'title', 'intro text')]
 
@@ -12,6 +13,12 @@ tile_images = [
     ('/static/template/images/tile-01.jpg',),
     ('/static/template/images/tile-02.jpg',)]
 
-rss_feed = [
-    ('/static/template/images/background.png', 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'), # simon ridley
-]
+#required (author,url)
+#optional (tags, image)
+rss_feeds = [
+    {'author':'Simon Ridley',
+     'url': 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'},
+    {'author':'Mathew Beddow', 'tags': ['tech'], 'url': 'http://www.matthewbeddow.co.uk/?feed=rss2'},
+    {'author':'Mike McRoberts', 'url': 'http://thearduinoguy.org/?feed=rss2'}]
+
+kent_hackspace = ['http://www.medwaymakers.co.uk/']

site/generate.py

@@ -13,6 +13,8 @@ from scaffold.web import www
 
 import constants as site
 
+import codecs
+
 from libs.rss_fetcher import feed_reader
 
 web = html()
@@ -39,28 +41,6 @@ def todict(data):
 def dict_to_list(data, keys):
     return [data.get(k) for k in keys]
 
-#~ class feed_reader:
-    #~ def __init__(self, url):
-        #~ self.feed = requests.get(url, stream=True)
-        #~ fp = open('rss_example.xml', 'r')
-        #~ self.feed = etree.parse(fp)
-        #~ self.feed = self.feed.getroot()
-#~ 
-        #~ self.title = self.feed.xpath('./channel/title/text()')[-1]
-        #~ self.link = self.feed.xpath('./channel/link/text()')[-1]
-        #~ self.description = self.feed.xpath('./channel/description/text()')[-1]
-#~ 
-        #~ self.channel_image = self.feed.xpath('.//image/url/text()')[-1]
-        #~ self.channel_image_title = self.feed.xpath('.//image/title/text()')[-1]
-        #~ self.channel_image_link = self.feed.xpath('.//image/link/text()')[-1]
-#~ 
-    #~ def __iter__(self):
-        #~ for item in self.feed.xpath('.//item'):
-            #~ title = item.xpath('./title/text()')
-            #~ link = item.xpath('./link/text()')
-            #~ description = item.xpath('./description/text()')
-            #~ yield title, link, description
-
 #~ class page:
     #~ def __enter__(self):
         #~ header()
@@ -102,29 +82,48 @@ def examples():
     web.tiles.create()
     #~ feed = feed_reader('')
     
-    feed_url = 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'
 
-    feed = feed_reader(feed_url)
+    feed = feed_reader(site.rss_feeds)
 
     for row in feed:
         print row
+        print type(row.get('description'))
         web.tiles.append(
             title = '%s By %s' %(row.get('title'), row.get('author')),
-            link = row.get('link'),
+            link = row.get('url'),
             image = row.get('image'), 
             description = row.get('description'))
-        web.div.append(str(row))
+        web.div.append(row)
     web.page.append(web.tiles.render())
 
     web.template.body.append(web.page.render())
     return footer()
 
+def blogs():
+    """ page for testing new components"""
+    header()
+    web.page.create('blogs')
+
+    web.tiles.create()
+    feed = feed_reader(site.rss_feeds)
+
+    for row in feed:
+        web.tiles.append(
+            title = row.get('title'),
+            author = row.get('author'),
+            link = row.get('url'),
+            image = row.get('image'), 
+            date = row.get('date'), 
+            description = row.get('description'))
+        web.div.append(row)
+    web.page.section(web.tiles.render())
+
+    web.template.body.append(web.page.render())
+    return footer()
+
 def index():
     header()
 
-    #~ web.menu.create('/', 'leftNav')
-    #~ web.menu * site.page_menu
-
     web.template.body.append(web.header_strip.create({}).render())
     web.template.body.append(web.menu.render())
 
@@ -136,6 +135,7 @@ def index():
             '/static/template/images/tile-01.jpg'
         ).set_classes('tile-right').render())
     web.banner_slider.reset()
+    print site.banner_images
     web.banner_slider * site.banner_images
     
     web.page.append(web.banner_slider.render())
@@ -180,8 +180,13 @@ if __name__ == "__main__":
     #~ args = parser.parse_args()
     #~ print(args.accumulate(args.integers))
 
-    with open('index.html', 'w') as fp:
-        fp.write(index())
-    with open('examples.html', 'w') as fp:
-        fp.write(examples())
+    with codecs.open('./index.html', 'w', "utf-8") as fp:
+        fp.write(index().decode('utf-8'))
+    #~ with open('./html/examples.html', 'w') as fp:
+        #~ fp.write(examples())
+    with codecs.open('./html/blog.html', 'w', "utf-8") as fp:
+        fp.write(blogs().decode('utf-8'))
 
+#~ file = codecs.open("lol", "w", "utf-8")
+#~ file.write(u'\ufeff')
+#~ file.close()

site/index.py

@@ -7,18 +7,25 @@ from flask import make_response
 
 import generate as pages
 
-app = Flask(__name__, static_url_path='/static')
 
-@app.route("/examples/", methods=['GET'])
+web_app = Flask(__name__, static_url_path='/static')
+
+# local testing server, add your pages here
+
+@web_app.route("/examples/", methods=['GET'])
 def examples():
+    """temporary for testing / examples"""
     return make_response(pages.examples())
+    
+@web_app.route("/blogs/", methods=['GET'])
+def blogs():
+    """temporary for testing / examples"""
+    return make_response(pages.blogs())
 
-@app.route("/", methods=['GET'])
+@web_app.route("/", methods=['GET'])
 def index():
+    """home page"""
     return make_response(pages.index())
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=True)
-
-
-#http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss
+    web_app.run(host='0.0.0.0', port=5000, debug=True)

site/libs/rss_fetcher.py

@@ -0,0 +1,157 @@
+import os
+import sys
+import lxml
+import pytz
+import datetime
+import requests
+import functools
+import requests.exceptions
+
+
+#from lxml import etree, objectify
+from lxml.html.clean import Cleaner
+
+namespaces = {
+    'atom': "http://www.w3.org/2005/Atom",
+    'openSearch': "http://a9.com/-/spec/opensearchrss/1.0/",
+    'blogger': "http://schemas.google.com/blogger/2008",
+    'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+    'slash': "http://purl.org/rss/1.0/modules/slash/",
+    'content': "http://purl.org/rss/1.0/modules/content/",
+    'taxo': "http://purl.org/rss/1.0/modules/taxonomy/",
+    'dc': "http://purl.org/dc/elements/1.1/",
+    'syn': "http://purl.org/rss/1.0/modules/syndication/",
+    'admin': "http://webns.net/mvcb/",
+    'feedburner': "http://rssnamespace.org/feedburner/ext/1.0",
+    'content': "http://purl.org/rss/1.0/modules/content/",
+    'wfw': "http://wellformedweb.org/CommentAPI/",
+    'dc': "http://purl.org/dc/elements/1.1/",
+    'atom': "http://www.w3.org/2005/Atom",
+    'sy': "http://purl.org/rss/1.0/modules/syndication/",
+    'slash': "http://purl.org/rss/1.0/modules/slash/"
+}
+
+
+#~ import zlib
+#~ 
+#~ READ_BLOCK_SIZE = 1024 * 8
+#~ def decompress_stream(fileobj):
+    #~ result = StringIO()
+#~ 
+    #~ d = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    #~ for chunk in iter(partial(response.raw.read, READ_BLOCK_SIZE), ''):
+        #~ result.write(d.decompress(chunk))
+#~ 
+    #~ result.seek(0)
+    #~ return result
+
+
+#~ parser = etree.XMLParser(remove_blank_text=True, ns_clean=True)
+#~ tree = etree.parse(metadata, parser)
+#~ root = tree.getroot()
+
+from email.utils import parsedate_tz, mktime_tz
+
+class feed_reader:
+    #create the html cleaner, this is to clean out unwanted html tags in the description text
+    html_cleaner = Cleaner()
+    html_cleaner.javascript = True 
+    html_cleaner.style = True
+    html_cleaner.remove_tags = ['script', 'iframe', 'link', 'style']
+    filter_by_date = datetime.datetime.now() - datetime.timedelta(days=int(1.5*365)) #  1 and a half years ago
+    #html_cleaner.allow_tags = ['script', 'iframe', 'link', 'style']
+    #html_cleaner.kill_tags = ['script', 'iframe', 'link', 'style']
+    
+    def __init__(self, feed_details, timeout=5):
+        self.results = {}
+        parser = lxml.etree.XMLParser(remove_blank_text=True, ns_clean=True, encoding='utf-8')
+        for feed_info in feed_details:
+            self.url = feed_info.get('url')
+            self.author = feed_info.get('author')
+            self.tags = feed_info.get('tags')
+            if feed_info.get('url').startswith('http:'):
+                response = requests.get(feed_info.get('url'), stream=True, timeout=timeout)
+                if response.headers.get('content-encoding') == 'gzip':
+                    response.raw.read = functools.partial(response.raw.read, decode_content=True)
+                self.feed = lxml.etree.parse(response.raw, parser)
+            else:
+                fp = open(feed_info.get('url'), 'r')
+                self.feed = lxml.etree.parse(fp, parser)
+            
+            self.feed = self.feed.getroot()
+            self.parse_feed()
+
+    def convert_rfc822_to_datetime(self, rfcdate):
+        if len(rfcdate):
+            parsed_rfcdate = parsedate_tz( rfcdate )
+            if not parsed_rfcdate:
+                return None
+            return datetime.datetime.fromtimestamp(
+                mktime_tz(parsed_rfcdate), pytz.utc ).replace(tzinfo=None)
+        return None
+
+    def clean_up_text(self, text):
+        """strip out any dirty tags like <script> they may break the sites"""
+        return self.html_cleaner.clean_html(text)
+
+    def fetch_node_text(self, node, name, default=''):
+        """fetch the text from the node we are given, we are working in unicode
+        so decode byte strings to unicode""" 
+        result = node.xpath('./%s' % name)
+        if result:
+            if type(result[-1].text) is str:
+                return result[-1].text.decode('utf8')
+            else:
+                return result[-1].text
+        else:
+            return default
+
+    def fetch_node_attribute(self, node, names, attribs, default):
+        result = node.xpath('./%s' % name)
+        if result:
+            return result.get(atrribs, '')
+        else:
+            return default
+
+    def format_author(self, author):
+        """extract the authors name from the author text node"""
+        return author.split('(')[-1].strip(')')
+
+    def filter(self, node, tags=None):
+        """filter the feed out by category tag, if no tags assume its pre filtered"""
+        if self.tags is None:
+            return True
+        for category in node.xpath('./category', namespaces=namespaces):
+            if category.text.lower() in self.tags:
+                return True
+        return False
+
+    def parse_feed(self):
+        """Parse the items in the feed, filter out bad data and put in defaults"""
+        for item in self.feed.xpath('.//item', namespaces=namespaces):
+            date = self.convert_rfc822_to_datetime(self.fetch_node_text(item, 'pubDate'))
+            if date > self.filter_by_date and self.filter(item):
+                self.filter(item)
+                self.results[date] = {
+                    'title': self.fetch_node_text(item, 'title'),
+                    'date': date,
+                    'url': self.fetch_node_text(item, 'link'),
+                    'author': self.format_author(self.fetch_node_text(item, 'author', self.author)),
+                    'image': self.fetch_node_text(item, 'image'),
+                    'description': self.clean_up_text(self.fetch_node_text(item, 'description'))}
+
+    def __iter__(self):
+        """return results ordered by date"""
+        for order in sorted(self.results.keys(), reverse=True):
+            #print str(self.results[order]['date']) + ' - ' + self.results[order]['author'] + ' - ' + self.results[order]['title']
+            yield self.results[order]
+        
+rss_feeds = [
+    {'author':'Simon Ridley', 'url': 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'},
+    {'author':'Mathew Beddow', 'tags': ['tech'], 'url': 'http://www.matthewbeddow.co.uk/?feed=rss2'},
+    {'author':'Mike McRoberts', 'url': 'http://thearduinoguy.org/?feed=rss2'}]
+
+#~ import .constants
+test = feed_reader(rss_feeds)
+for item in test:
+    pass

site/static/template/default.css

@@ -39,9 +39,10 @@ tbody tr:nth-child(odd) {
 /* new styles */
 .copyright {float:right;margin-top:20px;color:#fff;}
 
-p {margin:25px;}
+p {margin:25px;line-height:150%;}
 h2 {margin-left:25px;color:#fff;}
 h3 {color:#fff;}
+li {padding-bottom:10px;line-height:150%;}
 
 label {margin-bottom:10px;float:left;clear:both;display:block;color:#fff;}
 input {width:100%;margin:0px;margin-bottom:20px;padding:10px;}
@@ -57,24 +58,44 @@ button {margin-bottom:20px;background-color: #fff; height: 48px; width:100%; bor
 
 .menu {position:absolute;left:0px;right:0px;height:40px;height:388px;top:68px;background-color:#00232D;z-index:-1;}
 .menu ul {width:960px;margin:auto;height:40px;padding-left:20px;background-color:#0087A8;}
-.menu li {width:130px;float:left;list-style-type:none;margin:11px;text-align:center;}
+.menu li {width:130px;float:left;list-style-type:none;margin-top:10px;padding:00px;text-align:center;}
 .menu li:hover {width:130px;float:left;list-style-type:none;border-bottom:4px solid #1C4085;}
 .menu a {color:#ffffff;font-weight:bold;text-decoration:none;}
 .menu li:hover {}
 
 .page {width:980px;margin:auto;margin-top:155px;padding-bottom:48px;background-color:#0087A8;}
 
+.columns{float:right;clear:right;width:460px;margin-left:20px;margin-bottom:20px;}
+
 .google-groups-signup {width:300px;float:left;}
 .google-groups-signup input{width:274px;}
 .google-groups-signup a{color:#fff;}
 
 .banner-slide {position:absolute;width:750px;}
 .banner-slide ul{position:relative;overflow:hidden;width:750px;height:300px;margin:0px;}
-.banner-slide li{position:absolute;left:0px;top:0px;list-style-type:none;}
+.banner-slide li{position:absolute;left:0px;top:0px;list-style-type:none;padding:0px;margin:0px;}
+.banner-slide .button{background-color: #0087A8;border-radius:50%;opacity: 0.4;}
 .banner-slide .content{display:none;position:absolute;top:250px;width:100%;background-color: #eee;}
 .banner-slide .left{text-align:left;padding:6px;font-size:48px;color:#fff;position:absolute;top:122px;left:0px;width:60px;height:60px;background-color: transparent;}
 .banner-slide .right{text-align:right;padding:6px;font-size:48px;color:#fff;position:absolute;top:122px;right:0px;width:60px;height:60px;background-color: transparent;}
 
+.slide-button {
+  background-image: url('static/images/css/sprite-navigation-white.png');
+}
+
+.banner-slide .left {
+  background-position: -73px -39px;
+  width: 24px;
+  height: 24px;
+}
+
+.banner-slide .right {
+  background-position: -107px -39px;
+  width: 24px;
+  height: 24px;
+}
+
+
 /*slider animation*/
 .slide {position:absolute;left:0px;}
 
@@ -84,13 +105,25 @@ button {margin-bottom:20px;background-color: #fff; height: 48px; width:100%; bor
 .slide.ng-hide-remove {left:0px;opacity:0;}
 .slide.ng-hide-remove-active {transition:0.9s linear all;opacity:1;}
 
+.slide.ng-leave.ng-leave-active,
+.slide.ng-enter {
+    transition:0.5s linear all;
+    left:0px;opacity:0;
+}
+.slide.ng-leave,
+.slide.ng-enter.ng-enter-active {
+    transition:0.5s linear all;
+    left:0px;opacity:1;
 }
 
 .bullet-list li {margin:10px;}
 
-.tile {position:relative;background-color:#eee;width:460px;height:640px;margin-left:20px;margin-bottom:20px;float:left;box-shadow: 0px 2px 5px 0px rgba(0, 0, 0, 0.26);}
-.tile-img {width:100%;height:200px;overflow:hidden;background-repeat: no-repeat;background-position: center;background-image:url('/static/template/images/background.png');}
-.tile-content {position:absolute;bottom:20px;top:220px;overflow:scroll;left:20px;right:20px;text-align:justify;line-height:150%;font-size:12px;}
+.tile {position:relative;background-color:#eee;width:460px;margin-left:20px;margin-bottom:20px;float:left;box-shadow: 0px 2px 5px 0px rgba(0, 0, 0, 0.26);}
+.tile-img {margin:20px;width:420px;height:200px;overflow:hidden;background-repeat: no-repeat;background-position: center;background-image:url('/static/template/images/background.png');}
+.tile header {background-color: #00232D;width:100%;bottom:20px;top:220px;overflow:auto;left:20px;right:20px;text-align:justify;line-height:150%;font-size:12px;}
+.tile h2 {line-height:150%;font-size:12px;margin-right:20px;}
+.tile header a {color:#fff;}
+.tile-content {bottom:20px;top:220px;overflow:auto;left:20px;right:20px;text-align:justify;line-height:150%;font-size:12px;}
 .tile img {display:block;margin:auto;width:300px;height:200px;}
 
 #footertop {background-color:#00232D;height:48px;}

site/widgets/banner_slider.js

@@ -0,0 +1,41 @@
+var app = angular.module('myApp', ['ngAnimate']);
+
+app.controller('sliderController', function($scope, $interval) {
+    $scope.currentSlide = 0;
+    $scope.autoSlide = true;
+    //$scope.length = 0;
+
+    $scope.next = function() {
+        $scope.autoSlide=false;
+        if ($scope.currentSlide < $scope.length - 1){
+            $scope.currentSlide += 1;
+        }else{
+            $scope.currentSlide = 0;
+        };
+    }
+
+    $scope.prev = function() {
+        $scope.autoSlide=false;
+        if ($scope.currentSlide > 0){
+            $scope.currentSlide -= 1;
+        }else{
+            $scope.currentSlide = $scope.length - 1;
+        }
+    }
+
+    $scope.isCurrentSlideIndex = function (index) {
+        return $scope.currentSlide === index;
+    };
+
+    $scope.loopSlides = function (index) {
+        if ($scope.autoSlide==false){return false;}
+        if ($scope.currentSlide < $scope.length - 1){
+            $scope.currentSlide += 1;
+        }else{
+            $scope.currentSlide = 0;
+        }
+    };
+
+    $interval(function(){$scope.loopSlides();}, 5000);
+
+});

site/widgets/banner_slider.py

@@ -27,20 +27,21 @@ class control(www.default.html_ui):
         self.content=[]
 
     def append(self,image,link,title,intro=''):
-        htm='<a href="%s" ><img src="%s" /><div class="content">%s<br />%s</div></a>'%(link,image,title,intro)
+        htm = u'<a href="%s" ><img src="%s" /><div class="content">%s<br />%s</div></a>'%(link,image,title,intro)
         self.content.append(htm)
 
     def render(self):   
         #~ self.script.append(self.javascript())
         self.count+=1
-        htm='<div class="banner-slide" ng-app="myApp" ng-controller="sliderController">'
-        htm+='<ul style="%s" >' % self.height
-        count=0
-        #~ for item in self.content:
-            #~ htm+='<li class="slide" ng-hide="!isCurrentSlideIndex($index)">%s</li>' % (item)
-            #~ count+=1
-        htm += '''<li class="slide" ng-repeat="slide in slides" ng-hide="!isCurrentSlideIndex($index)" ng-show="isCurrentSlideIndex($index)"><a href="{{slide.link}}" ><img src="{{slide.src}}" /><div class="content">{{slide.title}}<br />{{slide.description}}</div></a></li>'''
-        htm += '<li style="clear:both;"></li></ul>'
-        htm += '<div ng-click="prev()" title="Previous" class="left">&lt;</div><div ng-click="next()" title="Next" class="right">&gt;</div>'
-        htm += '</div><div class="clear"></div>'
+        htm = u'<div class="banner-slide" ng-app="myApp" ng-controller="sliderController">'
+        htm += u'<ul style="%s" ng-switch on="currentSlide" ng-init="length=%d;">' % (self.height, len(self.content))
+        count = 0
+        for item in self.content:
+            htm += u'<li class="slide" ng-switch-when="%s">%s</li>' % (count, item)
+            count += 1
+        #htm += '''<li class="slide" ng-repeat="slide in slides" ng-hide="!isCurrentSlideIndex($index)" ng-show="isCurrentSlideIndex($index)"><a href="{{slide.link}}" ><img src="{{slide.src}}" /><div class="content">{{slide.title}}<br />{{slide.description}}</div></a></li>'''
+        htm += u'<li style="clear:both;"></li></ul>'
+        htm += u'<div ng-click="prev()" title="Previous" role="button" class="slide-button left">&lt;</div>'
+        htm += u'<div ng-click="next()" title="Next" role="button" class="slide-button right">&gt;</div>'
+        htm += u'</div><div class="clear"></div>'
         return htm

site/widgets/tiles.py

@@ -6,14 +6,25 @@ class control(www.default.html_ui):
         self.data = []
         return self
 
-    def append(self, title, link, image, description=''):
-        self.data.append((title, link, image, description))
+    def append(self, title, author, date, link, image, description=''):
+        self.data.append({
+        'title': title, 
+        'author': author,
+        'date': date,
+        'link': link,
+        'image': image, 
+        'description': description})
 
     def render(self):
-        htm = ''
+        htm = u''
         for project in self.data:
-            htm+='<div class="tile">'
-            htm+='<div class="tile-img"><img src="%s"/></div>' % project[2]
-            htm+='<div class="tile-content"><a href="%s">%s</a><p>%s</p></div>' % (project[1], project[0], project[3])
-            htm+='</div>'
+            htm += u'<div class="tile">'
+            if project.get('image'):
+                htm += u'<div class="tile-img"><img src="%s"/></div>' % project.get('image')
+            else:
+                htm += u'<div class="tile-img"></div>'
+            htm += u'<header class="tile-content"><h2><a href="%s">%s</a> By %s</h2></header>' % (
+                project.get('link'), project.get('title'),project.get('author'))
+            htm += u'<div class="tile-content"><p>%s</p></div>' % (project.get('description'))
+            htm += u'</div>'
         return htm