summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2010-11-13 09:54:55 +0100
committerYorhel <git@yorhel.nl>2010-11-13 09:54:55 +0100
commit935ad7c2c3b7f442664c84e2ccf44957dd642381 (patch)
treefa681c4e71f21f05be576fd9aeb21eefc9dca6d6
parentd48fef4c07d55a4983bfe6df49e72dae0c5d8b5e (diff)
Multi::Feed: Added Atom feeds
TODO: add links to these feeds from the site
-rw-r--r--.gitignore1
-rw-r--r--ChangeLog4
-rw-r--r--Makefile8
-rw-r--r--README6
-rw-r--r--data/global.pl1
-rw-r--r--lib/Multi/Feed.pm185
6 files changed, 200 insertions, 5 deletions
diff --git a/.gitignore b/.gitignore
index de859043..94b0eee6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
/data/docs/8
/data/log/
/static/f/script.js
+/static/feeds/
/static/s/*/style.css
/static/s/*/boxbg.png
/static/cv/
diff --git a/ChangeLog b/ChangeLog
index 49203056..6d2db866 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2.14 - ?
+ - Added Atom feeds for the recent announcements, changes and posts
+ (located in /www/feeds and updated every 15 min. by Multi::Feed)
+
2.13 - 2010-11-11
- Added 'formcode' parameter to all modification requests to fix all
cross-site request forgery vulnerabilities
diff --git a/Makefile b/Makefile
index af42c25d..d876d0c1 100644
--- a/Makefile
+++ b/Makefile
@@ -37,18 +37,18 @@
# environments. Patches to improve the portability are always welcome.
-.PHONY: all dirs js skins robots chmod chmod-tladmin multi-start multi-stop multi-restart sql-import update-2.10 update-2.11 update-2.12 update-2.13
+.PHONY: all dirs js skins robots chmod chmod-tladmin multi-start multi-stop multi-restart sql-import update-2.10 update-2.11 update-2.12 update-2.13 update-2.14
all: dirs js skins robots data/config.pl
-dirs: static/cv static/sf static/st data/log www
+dirs: static/cv static/sf static/st data/log www www/feeds
static/cv static/sf static/st:
mkdir $@;
for i in $$(seq -w 0 1 99); do mkdir "$@/$$i"; done
-data/log www:
+data/log www www/feeds:
mkdir $@
@@ -152,3 +152,5 @@ update-2.13: all
${runpsql} < util/updates/update_2.13.sql
$(multi-start)
+update-2.14: all
+
diff --git a/README b/README
index c63a08bb..c122f492 100644
--- a/README
+++ b/README
@@ -13,10 +13,10 @@ Requirements
global requirements:
Linux, or an OS that resembles Linux. Chances are VNDB won't run on Windows.
PostgreSQL 8.4+
- perl 5.10 recommended, 5.8 may also work
+ perl 5.12 recommended, 5.10 and 5.8 may also work
A webserver that works with YAWF (lighttpd and Apache are known to work)
- (perl 5.10 core modules are not listed.)
+ (perl 5.12 core modules are not listed.)
util/vndb.pl:
Algorithm::Diff::Fast
@@ -35,6 +35,8 @@ Requirements
DBD::Pg
POE
POE::Component::Pg (get it from http://g.blicky.net/poco-pg.git/)
+ Feed:
+ XML::Writer
IRC:
POE::Component::IRC
URI::Escape
diff --git a/data/global.pl b/data/global.pl
index 0e7b6d8c..cc251851 100644
--- a/data/global.pl
+++ b/data/global.pl
@@ -109,6 +109,7 @@ our %M = (
log_dir => $ROOT.'/data/log',
modules => {
#API => {}, # disabled by default, not really needed
+ Feed => {},
RG => {},
Image => {},
#Anime => {}, # disabled by default, requires AniDB username/pass
diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm
new file mode 100644
index 00000000..4c157b2d
--- /dev/null
+++ b/lib/Multi/Feed.pm
@@ -0,0 +1,185 @@
+
+#
+# Multi::Feed - Generates and updates Atom feeds
+#
+
+package Multi::Feed;
+
+use strict;
+use warnings;
+use POE;
+use XML::Writer;
+use POSIX 'strftime';
+use Time::HiRes 'time';
+use VNDBUtil 'bb2html';
+
+
+sub spawn {
+ my $p = shift;
+ POE::Session->create(
+ package_states => [
+ $p => [qw| _start shutdown generate write_atom log_stats |],
+ ],
+ heap => {
+ regenerate_interval => 900, # 15 min.
+ stats_interval => 86400, # daily
+ num_announcements => 10,
+ num_changes => 25,
+ num_posts => 25,
+ debug => 0,
+ @_,
+ stats => {}, # key = feed, value = [ count, total, max ]
+ },
+ );
+}
+
+
+sub _start {
+ $_[KERNEL]->alias_set('feed');
+ $_[KERNEL]->yield('generate');
+ $_[KERNEL]->alarm(log_stats => int((time+3)/$_[HEAP]{stats_interval}+1)*$_[HEAP]{stats_interval});
+ $_[KERNEL]->sig(shutdown => 'shutdown');
+}
+
+
+sub shutdown {
+ $_[KERNEL]->delay('generate');
+ $_[KERNEL]->delay('log_stats');
+ $_[KERNEL]->alias_remove('feed');
+}
+
+
+sub generate {
+ $_[KERNEL]->alarm(generate => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval});
+
+ # announcements
+ $_[KERNEL]->post(pg => query => q{
+ SELECT '/t'||t.id AS id, t.title, extract('epoch' from tp.date) AS published,
+ extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
+ FROM threads t
+ JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1
+ JOIN threads_boards tb ON tb.tid = t.id AND tb.type = 'an'
+ JOIN users u ON u.id = tp.uid
+ WHERE NOT t.hidden
+ ORDER BY t.id DESC
+ LIMIT ?}, [ $_[HEAP]{num_announcements} ], 'write_atom',
+ {
+ feed => 'announcements',
+ title => 'VNDB.org Site Announcements',
+ id => '/t/an',
+ });
+
+ # changes
+ $_[KERNEL]->post(pg => query => q{
+ SELECT '/'||c.type||COALESCE(vr.vid, rr.rid, pr.pid)||'.'||c.rev AS id,
+ COALESCE(vr.title, rr.title, pr.name) AS title, extract('epoch' from c.added) AS updated,
+ u.username, u.id AS uid, c.comments AS summary
+ FROM changes c
+ LEFT JOIN vn_rev vr ON c.type = 'v' AND c.id = vr.id
+ LEFT JOIN releases_rev rr ON c.type = 'r' AND c.id = rr.id
+ LEFT JOIN producers_rev pr ON c.type = 'p' AND c.id = pr.id
+ JOIN users u ON u.id = c.requester
+ WHERE c.requester <> 1
+ ORDER BY c.id DESC
+ LIMIT ?}, [ $_[HEAP]{num_changes} ], 'write_atom',
+ {
+ feed => 'changes',
+ title => 'VNDB.org Recent Changes',
+ id => '/hist',
+ });
+
+ # posts (this query isn't all that fast)
+ $_[KERNEL]->post(pg => query => q{
+ SELECT '/t'||t.id||'.'||tp.num AS id, t.title||' (#'||tp.num||')' AS title, extract('epoch' from tp.date) AS published,
+ extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
+ FROM threads_posts tp
+ JOIN threads t ON t.id = tp.tid
+ JOIN users u ON u.id = tp.uid
+ WHERE NOT tp.hidden AND NOT t.hidden
+ ORDER BY tp.date DESC
+ LIMIT ?}, [ $_[HEAP]{num_posts} ], 'write_atom',
+ {
+ feed => 'posts',
+ title => 'VNDB.org Recent Posts',
+ id => '/t',
+ });
+}
+
+
+sub write_atom { # num, res, nfo, time
+ my $r = $_[ARG1];
+ my $nfo = $_[ARG2];
+
+ my $start = time;
+
+ my $updated = 0;
+ for(@$r) {
+ $updated = $_->{published} if $_->{published} && $_->{published} > $updated;
+ $updated = $_->{updated} if $_->{updated} && $_->{updated} > $updated;
+ }
+
+ my $data;
+ my $x = XML::Writer->new(OUTPUT => \$data, DATA_MODE => 1, DATA_INDENT => 2);
+ $x->xmlDecl('UTF-8');
+ $x->startTag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => $VNDB::S{url}.'/');
+ $x->dataElement(title => $nfo->{title});
+ $x->dataElement(updated => datetime($updated));
+ $x->dataElement(id => $VNDB::S{url}.$nfo->{id});
+ $x->emptyTag(link => rel => 'self', type => 'application/atom+xml', href => "$VNDB::S{url}/feeds/$nfo->{feed}.atom");
+ $x->emptyTag(link => rel => 'alternate', type => 'text/html', href => $nfo->{id});
+
+ for(@$r) {
+ $x->startTag('entry');
+ $x->dataElement(id => $VNDB::S{url}.$_->{id});
+ $x->dataElement(title => $_->{title});
+ $x->dataElement(updated => $_->{updated}?datetime($_->{updated}):datetime($_->{published}));
+ $x->dataElement(published => datetime($_->{published})) if $_->{published};
+ if($_->{username}) {
+ $x->startTag('author');
+ $x->dataElement(name => $_->{username});
+ $x->dataElement(uri => '/u'.$_->{uid}) if $_->{uid};
+ $x->endTag('author');
+ }
+ $x->emptyTag(link => rel => 'alternate', type => 'text/html', href => $_->{id});
+ $x->dataElement(summary => bb2html($_->{summary}, 200), type => 'html') if $_->{summary};
+ $x->endTag('entry');
+ }
+
+ $x->endTag('feed');
+
+ open my $f, '>:utf8', "$VNDB::ROOT/www/feeds/$nfo->{feed}.atom" || die $!;
+ print $f $data;
+ close $f;
+
+ $_[HEAP]{debug} && $_[KERNEL]->call(core => log => 'Wrote %s.atom (%d entries, sql:%4dms, perl:%4dms)',
+ $nfo->{feed}, scalar(@$r), $_[ARG3]*1000, (time-$start)*1000);
+
+ $_[HEAP]{stats}{$nfo->{feed}} = [ 0, 0, 0 ] if !$_[HEAP]{stats}{$nfo->{feed}};
+ my $time = ((time-$start)+$_[ARG3])*1000;
+ $_[HEAP]{stats}{$nfo->{feed}}[0]++;
+ $_[HEAP]{stats}{$nfo->{feed}}[1] += $time;
+ $_[HEAP]{stats}{$nfo->{feed}}[2] = $time if $_[HEAP]{stats}{$nfo->{feed}}[2] < $time;
+}
+
+
+sub log_stats {
+ $_[KERNEL]->alarm(log_stats => int((time+3)/$_[HEAP]{stats_interval}+1)*$_[HEAP]{stats_interval});
+
+ for (keys %{$_[HEAP]{stats}}) {
+ my $v = $_[HEAP]{stats}{$_};
+ next if !$v->[0];
+ $_[KERNEL]->call(core => log => 'Stats summary for %s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB',
+ $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s "$VNDB::ROOT/www/feeds/$_.atom")/1024);
+ }
+ $_[HEAP]{stats} = {};
+}
+
+
+# non-POE helper function
+sub datetime {
+ strftime('%Y-%m-%dT%H:%M:%SZ', gmtime shift);
+}
+
+
+1;
+