158 lines
6.2 KiB
Python
158 lines
6.2 KiB
Python
import os
|
|
import sys
|
|
import lxml
|
|
import pytz
|
|
import datetime
|
|
import requests
|
|
import functools
|
|
import requests.exceptions
|
|
|
|
|
|
#from lxml import etree, objectify
|
|
from lxml.html.clean import Cleaner
|
|
|
|
namespaces = {
|
|
'atom': "http://www.w3.org/2005/Atom",
|
|
'openSearch': "http://a9.com/-/spec/opensearchrss/1.0/",
|
|
'blogger': "http://schemas.google.com/blogger/2008",
|
|
'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
'slash': "http://purl.org/rss/1.0/modules/slash/",
|
|
'content': "http://purl.org/rss/1.0/modules/content/",
|
|
'taxo': "http://purl.org/rss/1.0/modules/taxonomy/",
|
|
'dc': "http://purl.org/dc/elements/1.1/",
|
|
'syn': "http://purl.org/rss/1.0/modules/syndication/",
|
|
'admin': "http://webns.net/mvcb/",
|
|
'feedburner': "http://rssnamespace.org/feedburner/ext/1.0",
|
|
'content': "http://purl.org/rss/1.0/modules/content/",
|
|
'wfw': "http://wellformedweb.org/CommentAPI/",
|
|
'dc': "http://purl.org/dc/elements/1.1/",
|
|
'atom': "http://www.w3.org/2005/Atom",
|
|
'sy': "http://purl.org/rss/1.0/modules/syndication/",
|
|
'slash': "http://purl.org/rss/1.0/modules/slash/"
|
|
}
|
|
|
|
|
|
#~ import zlib
|
|
#~
|
|
#~ READ_BLOCK_SIZE = 1024 * 8
|
|
#~ def decompress_stream(fileobj):
|
|
#~ result = StringIO()
|
|
#~
|
|
#~ d = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
|
#~ for chunk in iter(partial(response.raw.read, READ_BLOCK_SIZE), ''):
|
|
#~ result.write(d.decompress(chunk))
|
|
#~
|
|
#~ result.seek(0)
|
|
#~ return result
|
|
|
|
|
|
#~ parser = etree.XMLParser(remove_blank_text=True, ns_clean=True)
|
|
#~ tree = etree.parse(metadata, parser)
|
|
#~ root = tree.getroot()
|
|
|
|
from email.utils import parsedate_tz, mktime_tz
|
|
|
|
class feed_reader:
|
|
#create the html cleaner, this is to clean out unwanted html tags in the description text
|
|
html_cleaner = Cleaner()
|
|
html_cleaner.javascript = True
|
|
html_cleaner.style = True
|
|
html_cleaner.remove_tags = ['script', 'iframe', 'link', 'style']
|
|
filter_by_date = datetime.datetime.now() - datetime.timedelta(days=int(1.5*365)) # 1 and a half years ago
|
|
#html_cleaner.allow_tags = ['script', 'iframe', 'link', 'style']
|
|
#html_cleaner.kill_tags = ['script', 'iframe', 'link', 'style']
|
|
|
|
def __init__(self, feed_details, timeout=5):
|
|
self.results = {}
|
|
parser = lxml.etree.XMLParser(remove_blank_text=True, ns_clean=True, encoding='utf-8')
|
|
for feed_info in feed_details:
|
|
self.url = feed_info.get('url')
|
|
self.author = feed_info.get('author')
|
|
self.tags = feed_info.get('tags')
|
|
if feed_info.get('url').startswith('http:'):
|
|
response = requests.get(feed_info.get('url'), stream=True, timeout=timeout)
|
|
if response.headers.get('content-encoding') == 'gzip':
|
|
response.raw.read = functools.partial(response.raw.read, decode_content=True)
|
|
self.feed = lxml.etree.parse(response.raw, parser)
|
|
else:
|
|
fp = open(feed_info.get('url'), 'r')
|
|
self.feed = lxml.etree.parse(fp, parser)
|
|
|
|
self.feed = self.feed.getroot()
|
|
self.parse_feed()
|
|
|
|
def convert_rfc822_to_datetime(self, rfcdate):
|
|
if len(rfcdate):
|
|
parsed_rfcdate = parsedate_tz( rfcdate )
|
|
if not parsed_rfcdate:
|
|
return None
|
|
return datetime.datetime.fromtimestamp(
|
|
mktime_tz(parsed_rfcdate), pytz.utc ).replace(tzinfo=None)
|
|
return None
|
|
|
|
def clean_up_text(self, text):
|
|
"""strip out any dirty tags like <script> they may break the sites"""
|
|
return self.html_cleaner.clean_html(text)
|
|
|
|
def fetch_node_text(self, node, name, default=''):
|
|
"""fetch the text from the node we are given, we are working in unicode
|
|
so decode byte strings to unicode"""
|
|
result = node.xpath('./%s' % name)
|
|
if result:
|
|
if type(result[-1].text) is str:
|
|
return result[-1].text.decode('utf8')
|
|
else:
|
|
return result[-1].text
|
|
else:
|
|
return default
|
|
|
|
def fetch_node_attribute(self, node, names, attribs, default):
|
|
result = node.xpath('./%s' % name)
|
|
if result:
|
|
return result.get(atrribs, '')
|
|
else:
|
|
return default
|
|
|
|
def format_author(self, author):
|
|
"""extract the authors name from the author text node"""
|
|
return author.split('(')[-1].strip(')')
|
|
|
|
def filter(self, node, tags=None):
|
|
"""filter the feed out by category tag, if no tags assume its pre filtered"""
|
|
if self.tags is None:
|
|
return True
|
|
for category in node.xpath('./category', namespaces=namespaces):
|
|
if category.text.lower() in self.tags:
|
|
return True
|
|
return False
|
|
|
|
def parse_feed(self):
|
|
"""Parse the items in the feed, filter out bad data and put in defaults"""
|
|
for item in self.feed.xpath('.//item', namespaces=namespaces):
|
|
date = self.convert_rfc822_to_datetime(self.fetch_node_text(item, 'pubDate'))
|
|
if date > self.filter_by_date and self.filter(item):
|
|
self.filter(item)
|
|
self.results[date] = {
|
|
'title': self.fetch_node_text(item, 'title'),
|
|
'date': date,
|
|
'url': self.fetch_node_text(item, 'link'),
|
|
'author': self.format_author(self.fetch_node_text(item, 'author', self.author)),
|
|
'image': self.fetch_node_text(item, 'image'),
|
|
'description': self.clean_up_text(self.fetch_node_text(item, 'description'))}
|
|
|
|
def __iter__(self):
|
|
"""return results ordered by date"""
|
|
for order in sorted(self.results.keys(), reverse=True):
|
|
#print str(self.results[order]['date']) + ' - ' + self.results[order]['author'] + ' - ' + self.results[order]['title']
|
|
yield self.results[order]
|
|
|
|
rss_feeds = [
|
|
{'author':'Simon Ridley', 'url': 'http://waistcoatforensicator.blogspot.com/feeds/posts/default?alt=rss'},
|
|
{'author':'Mathew Beddow', 'tags': ['tech'], 'url': 'http://www.matthewbeddow.co.uk/?feed=rss2'},
|
|
{'author':'Mike McRoberts', 'url': 'http://thearduinoguy.org/?feed=rss2'}]
|
|
|
|
#~ import .constants
|
|
test = feed_reader(rss_feeds)
|
|
for item in test:
|
|
pass
|