コミット: 2244893f997089e97f35e075de10ccad4cb33c5f - newsfour (git)

news4 - RSS aggrigation system

コミットメタ情報

• リビジョン: 2244893f997089e97f35e075de10ccad4cb33c5f (tree)
• 日時: 2012-11-07 05:33:20
• 作者: hylom <hylom@hylo...>
• コミッター: hylom

ログメッセージ

Merge branch 'master' into live

変更サマリ

• modified: README
• modified: config.py.sample
• add: config2ini.py
• add: configloader.py
• modified: css/gnews.css
• modified: fetcher.py
• add: filters/remove_tracker.py
• modified: gnews.py
• modified: logger.py
• add: plugins/hatebu_counter.py
• modified: renderer.py
• add: sources.ini.sample
• modified: templates/index.tmpl.html

差分

--- a/README
+++ b/README
@@ -14,9 +14,12 @@
 　gnewsでサイトを生成するために、下記の設定ファイルが必要です。
 
 === config.py ===
-　HTMLの生成先やサイト名、RSSの取得先などを設定するファイルです。PythonのDictionaryおよびArray形式で記述されています。
+　HTMLの生成先やサイト名などを設定するファイルです。PythonのDictionaryおよびArray形式で記述されています。
 　config.py.sampleをコピーしてconfig.pyを作成し、編集します。
 
+=== sources.ini ===
+　RSSの取得先を設定するファイルです。ini形式で記述されています。セクション名が表示されるサイト名、urlパラメータがサイトのURL、sourcesパラメータがRSSの取得先、filtersが使用するフィルタ一覧（カンマ区切り）となります。
+
 === install.conf ===
 　関連ファイルのコピー先を指定するファイルです。
 　install.conf.sampleをコピーしてintall.confを作成し、編集します。

--- a/config.py.sample
+++ b/config.py.sample
@@ -7,6 +7,7 @@ config = {
     'output_directory': 'outputs',
     'filter_directory': 'filters',
     'pagination_unit': 20,
+    'log_level': 0,
     'index': {
         'template': 'index.tmpl.html',
         'output_directory': 'outputs/',
@@ -30,59 +31,7 @@ config = {
     'post_filters': [
         'cleanup',
         'trimming',
+        'remove_tracker'
         ],
 }
 
-target_rss = [
-    {
-        'name': 'SourceForge.JP Magazine',
-        'url': 'http://rss.rssad.jp/rss/sourceforge/magazine/rss',
-        'source_url': 'http://sourceforge.jp/magazine/',
-     },
-    {
-        'name': 'Slashdot Japan',
-        'url': 'http://rss.rssad.jp/rss/slashdot/slashdot.rss',
-        'source_url': 'http://slashdot.jp/',
-		'filter': ['slashdotjp',],
-     },
-    {
-        'name': 'ITmedia',
-        'url': 'http://rss.rssad.jp/rss/itmtop/2.0/itmedia_all.xml',
-        'source_url': 'http://www.itmedia.co.jp/',
-        'filter': ['tagging', 'itmedia'],
-     },
-    {
-        'name': 'So-netセキュリティ通信',
-        'url': 'http://security-t.blog.so-net.ne.jp/index.xml',
-        'source_url': 'http://security-t.blog.so-net.ne.jp/',
-        'filter': ['tagging',],
-     },
-    {
-        'name': 'Engadget Japanese',
-        'url': 'http://japanese.engadget.com/rss.xml',
-        'source_url': 'http://japanese.engadget.com/',
-        'filter': ['tagging',],
-     },
-    {
-        'name': 'ギズモード・ジャパン',
-        'url': 'http://feeds.gizmodo.jp/rss/gizmodo/index.xml',
-        'source_url': 'http://www.gizmodo.jp/',
-        'filter': ['tagging',],
-     },
-    {
-        'name': 'TechCrunch Japan',
-        'url': 'http://jp.techcrunch.com/feed/',
-        'source_url': 'http://jp.techcrunch.com/',
-        'filter': ['tagging',],
-     },
-    ]
-
-
-"""Template:
-    {
-        'name': '',
-        'url': '',
-        'source_url': '',
-        'filter': ['tagging',],
-     },
-"""

--- /dev/null
+++ b/config2ini.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python
+
+from config import config as config, target_rss as target_rss
+import ConfigParser
+import sys
+
+def main():
+    config = ConfigParser.SafeConfigParser()
+    for item in target_rss:
+        config.add_section(item["name"])
+        config.set(item["name"], 'source', item["url"])
+        config.set(item["name"], 'url', item["source_url"])
+        if 'filter' in item:
+            filters = ",".join(item["filter"])
+            config.set(item["name"], 'filters', filters)
+    config.write(sys.stdout)
+
+if __name__ == '__main__':
+    main()
+
+

--- /dev/null
+++ b/configloader.py
@@ -0,0 +1,27 @@
+# configloader.py
+# -*- config: utf-8 -*-
+
+import ConfigParser
+
+CONFIG_FILE = 'sources.ini'
+
+def load():
+    'parse .ini file and create config object'
+    config = ConfigParser.SafeConfigParser()
+    fp = open(CONFIG_FILE, 'r')
+    config.readfp(fp)
+    fp.close()
+    sources = []
+    for section in config.sections():
+        source = {}
+        source["name"] = section
+        source["source"] = config.get(section, 'source')
+        source["url"] = config.get(section, 'url')
+        if config.has_option(section, 'filters'):
+            filters = config.get(section, 'filters').split(',')
+            filters = [x.strip() for x in filters]
+            source["filters"] = filters
+        sources.append(source)
+    return sources
+
+            

--- a/css/gnews.css
+++ b/css/gnews.css
@@ -10,21 +10,33 @@ a {
   margin-bottom: 1em;
 }
 
+.entry-header .thumbnail {
+    width: 100px;
+    float: right;
+    margin-left: 10px;
+    margin-bottom: 10px;
+    pagging: 1px;
+}
+
 .entry-continue {
   margin-bottom: 1em;
 }
 .entry-footer{
-  color: gray;
+  color: #888;
 }
 
 #site-header {
-  border-bottom: 1px solid gray;
+  border-bottom: 1px solid #888;
   margin-bottom: 10px;
 }
 
+#site-header .last-update {
+  color: #888;
+}
+
 #site-footer {
   margin-top: 10px;
-  color: gray;
+  color: #888;
   text-align: center;
 }
 

--- a/fetcher.py
+++ b/fetcher.py
@@ -5,7 +5,7 @@ import re
 
 import feedparser
 import dateutil.parser
-from config import config as config, target_rss as target_rss
+from config import config as config
 from logger import log
 
 class FeedFetcher(object):
@@ -15,13 +15,13 @@ class FeedFetcher(object):
 
     def _fetch(self):
         'do fetch'
-        f = feedparser.parse(self._feed["url"])
+        f = feedparser.parse(self._feed["source"])
         entries = []
         for e in f['entries']:
             entry = {
 #                'title': e.title.decode('utf8') if isinstance(e.title, str) else e.title,
                 'title': e.title,
-                'link': e.link,
+                'url': e.link,
                 'body': e.description,
                 'date': dateutil.parser.parse(e.updated),
                 'feed': self._feed,
@@ -58,8 +58,8 @@ class FeedFetcher(object):
         entries = self._fetch()
         entries = self._apply_pre_filters(entries)
 
-        if 'filter' in self._feed:
-            filters = self._feed.get('filter', None)
+        if 'filters' in self._feed:
+            filters = self._feed.get('filters', None)
             entries = self._apply_filters(filters, entries)
 
         entries = self._apply_post_filters(entries)

--- /dev/null
+++ b/filters/remove_tracker.py
@@ -0,0 +1,15 @@
+# remove images for tracking
+# -*- coding: utf-8 -*-
+
+import re
+
+re_rssad_url = re.compile(r'^http://rss.rssad.jp/')
+
+def entry_filter(entry):
+    if "images" in entry:
+        for i in range(len(entry["images"])):
+            if re_rssad_url.search(entry["images"][i]):
+                entry["images"].pop(i)
+
+    return entry
+

--- a/gnews.py
+++ b/gnews.py
@@ -1,20 +1,23 @@
 #!/usr/bin/python
 'gnews.py - google news clone'
 
-from config import config as config, target_rss as target_rss
+from config import config as config
 import renderer
 import fetcher
 import os.path
 import urllib
 from logger import log
+import configloader
+
+sources = configloader.load()
 
 def main():
     "gnews's main function"
-    # TODO: argv check
+# TODO: argv check
 
     # fetch RSS feed
     entries = []
-    for feed in target_rss:
+    for feed in sources:
         f = fetcher.FeedFetcher(feed)
         e = f.get_entries()
         entries.extend(e)
@@ -45,8 +48,14 @@ def main():
     for e in entries:
         log(e["date"])
 
+    call_plugin('pre_render', entries)
+
     # do rendering
-    params = {'tags':tags, 'page':{}, 'sorted_tags':sorted_tags}
+    params = {
+        'tags':tags,
+        'page':{},
+        'sorted_tags':sorted_tags
+        }
 
     # render index page
     do_rendering('index', 'index%s.html', entries, params)
@@ -57,11 +66,43 @@ def main():
         do_rendering('tags', tag + '%s.html', subentries, params)
 
 
+def call_plugin(function_name, entries):
+    "call plugin"
+    for plugin in config['plugins']:
+        mod = _get_plugin(plugin)
+        f = mod.__getattribute__(function_name)
+        f(entries)
+
+class PluginError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return 'plugin "' + self.value + '" is not found.'
+
+def _get_plugin(plugin_name):
+    'load plugin by config settings'
+
+    # fallback when filter isn't defined
+    if plugin_name is None:
+        return lambda x:x
+        
+    # import module
+    mods = __import__(config['plugin_directory'], 
+                      globals(),
+                      locals(),
+                      [plugin_name,])
+    try:
+        mod = mods.__getattribute__(plugin_name)
+    except AttributeError:
+        raise PluginError(plugin_name)
+
+    return mod
+
 
 def do_rendering(page_type, filename, entries, params):
     "rendering page"
 
-    r = renderer.Renderer()
+    r = renderer.Renderer(sources)
     tmpl = config[page_type]['template']
     output_dir = config[page_type]['output_directory']
 

--- a/logger.py
+++ b/logger.py
@@ -1,6 +1,6 @@
 'logger.py - log output utility'
 
-from config import config, target_rss
+from config import config
 
 def log(*args):
     "log helper function"

--- /dev/null
+++ b/plugins/hatebu_counter.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python
+# -*- coding: utf-8
+'plugin for hatena bookmark counter'
+
+#from __future__ import with_statement
+
+import xmlrpclib
+import datetime
+import time
+import sys
+
+# see http://d.hatena.ne.jp/keyword/%a4%cf%a4%c6%a4%ca%a5%d6%a5%c3%a5%af%a5%de%a1%bc%a5%af%b7%ef%bf%f4%bc%e8%c6%c0API?kid=146686
+
+urls = []
+counts = {}
+
+def pre_fetch():
+    pass
+
+def pre_tag_aggregate(entries):
+    pass
+
+def pre_render(entries):
+    counts = []
+    for i in range(0, len(entries), 50):
+        urls = [x['url'] for x in entries[i:i+50]]
+        c = _get_count(urls)
+        for j in range(0, len(c)):
+            entries[i+j]['url'] = c[j]
+
+def pre_quit(entries):
+    pass
+
+def _get_count(urls):    
+    # urls can have max 50 items
+    uri = "http://b.hatena.ne.jp/xmlrpc"
+    server = xmlrpclib.ServerProxy(uri)
+    t = server.bookmark.getCount(*urls)
+    return t
+

--- a/renderer.py
+++ b/renderer.py
@@ -7,17 +7,17 @@ from mako.lookup import TemplateLookup
 from mako.exceptions import RichTraceback
 import dateutil.parser
 
-from config import config, target_rss
+from config import config
 from propertizer import propertize
 from logger import log
 
 def date_format(date):
-    #dt = dateutil.parser.parse(date)
     return date.strftime('%Y/%m/%d %H:%M')
 
 class Renderer(object):
-    def __init__(self):
+    def __init__(self, sources):
         self.template_dir = config['template_directory']
+        self._sources = sources
         
     def _get_template(self, template_name):
         'read template file'
@@ -38,8 +38,9 @@ class Renderer(object):
             'entries': entries,
             'params': params,
             'site': config['site_parameter'],
-            'targets': target_rss,
+            'sources': self._sources,
             }
+        kwargs['site']['last_update'] = datetime.datetime.utcnow()
         for key in kwargs:
             d = propertize(kwargs[key])
             kwargs[key] = d

--- /dev/null
+++ b/sources.ini.sample
@@ -0,0 +1,54 @@
+[So-netセキュリティ通信]
+url = http://security-t.blog.so-net.ne.jp/
+source = http://security-t.blog.so-net.ne.jp/index.rdf
+filters = tagging
+
+[Impress Watch]
+url = http://www.watch.impress.co.jp/
+source = http://rss.rssad.jp/rss/headline/headline.rdf
+filters = tagging
+
+[japan.internet.com]
+url = http://japan.internet.com/
+source = http://rss.internetcom.jp/rss/japaninternetcom/index.rdf
+filters = tagging
+
+[SourceForge.JP Magazine]
+url = http://sourceforge.jp/magazine/
+source = http://rss.rssad.jp/rss/sourceforge/magazine/rss
+
+[WIRED.jp]
+url = http://wired.jp/
+source = http://rss.rssad.jp/rss/h/wired/feed.rdf
+filters = tagging
+
+[CNET Japan]
+url = http://japan.cnet.com/
+source = http://feeds.japan.cnet.com/rss/cnet/all.rdf
+filters = tagging
+
+[TechCrunch Japan]
+url = http://jp.techcrunch.com/
+source = http://jp.techcrunch.com/feed/
+filters = tagging
+
+[ギズモード・ジャパン]
+url = http://www.gizmodo.jp/
+source = http://feeds.gizmodo.jp/rss/gizmodo/index.xml
+filters = tagging
+
+[Slashdot Japan]
+url = http://slashdot.jp/
+source = http://rss.rssad.jp/rss/slashdot/slashdot.rss
+filters = slashdotjp
+
+[Engadget Japanese]
+url = http://japanese.engadget.com/
+source = http://japanese.engadget.com/rss.xml
+filters = tagging
+
+[ITmedia]
+url = http://www.itmedia.co.jp/
+source = http://rss.rssad.jp/rss/itmtop/2.0/itmedia_all.xml
+filters = tagging,itmedia
+

--- a/templates/index.tmpl.html
+++ b/templates/index.tmpl.html
@@ -25,14 +25,17 @@ s.parentNode.insertBefore(ga, s);
     <div class="container">
 
       <!-- タイトル -->
-      <div class="row">
-        <div class="span12">
-          <header id="site-header">
+      <div class="row" id="site-header">
+        <div class="span9">
+          <header>
             <a href="${site.root}">
               <img id="sitelogo" src="${site.img_directory}/themesjp.png" alt="Themes.JP"> α
             </a>
           </header>
         </div>
+        <div  class="span3 last-update">
+          last update: ${date_format(site.last_update)}
+        </div>
       </div>
 
       <!-- コンテンツ本体 -->
@@ -59,33 +62,32 @@ s.parentNode.insertBefore(ga, s);
           <div class="entry">
             <!-- ヘッダ -->
             <div class="entry-header">
+              % if 'images' in entry and len(entry.images) > 0:
+              <img class="thumbnail" src="${entry.images[0]}">
+              % endif
               <h3>
-                <a href='${entry.link}'  target="_blank_">${entry.title}</a>
+                <a href='${entry.url}'  target="_blank_">${entry.title}</a>
               </h3>
             </div>
 
 
             <!-- 本文テキスト -->
-            <div class="entry-body">${entry.body}</div>
+            <div class="entry-body">
+              ${entry.body}
+            </div>
 
             <!-- フッタ -->
             <div class="entry-footer">
               <div class="entry-continue">
-                <a href='${entry.link}'>［続きを読む］</a>
+                <a href='${entry.url}'>［続きを読む］</a>
               </div>
               <div class="information">
-                <span>情報元：<a href='${entry.feed.source_url}'>${entry.feed.name}</a></span>
+                <span>情報元：<a href='${entry.feed.url}'>${entry.feed.name}</a></span>
                 <span>（${date_format(entry.date)}）</span>
                 <span>タグ：</span>
                 % for tag in entry.tags:
                 <span>${tag} </span>
                 % endfor
-                % if 'images' in entry:
-                <span>画像：</span>
-                % for imgurl in entry.images:
-                <span><a href="${imgurl}">*</a></span>
-                % endfor
-                % endif
               </div>
             </div>
           </div>
@@ -115,8 +117,8 @@ s.parentNode.insertBefore(ga, s);
           <div class="feed-provider">
             <h3>情報提供サイト:</h3>
             <ul class="nav nav-pills nav-stacked">
-              % for item in targets:
-              <li><a href="${item.source_url}">${item.name}</a></li>
+              % for item in sources:
+              <li><a href="${item.source}">${item.name}</a></li>
               % endfor
             </ul>
           </div>