diff options
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | ChangeLog | 4 | ||||
-rw-r--r-- | Makefile | 8 | ||||
-rw-r--r-- | README | 6 | ||||
-rw-r--r-- | data/global.pl | 1 | ||||
-rw-r--r-- | lib/Multi/Feed.pm | 185 |
6 files changed, 200 insertions, 5 deletions
@@ -3,6 +3,7 @@ /data/docs/8 /data/log/ /static/f/script.js +/static/feeds/ /static/s/*/style.css /static/s/*/boxbg.png /static/cv/ @@ -1,3 +1,7 @@ +2.14 - ? + - Added Atom feeds for the recent announcements, changes and posts + (located in /www/feeds and updated every 15 min. by Multi::Feed) + 2.13 - 2010-11-11 - Added 'formcode' parameter to all modification requests to fix all cross-site request forgery vulnerabilities @@ -37,18 +37,18 @@ # environments. Patches to improve the portability are always welcome. -.PHONY: all dirs js skins robots chmod chmod-tladmin multi-start multi-stop multi-restart sql-import update-2.10 update-2.11 update-2.12 update-2.13 +.PHONY: all dirs js skins robots chmod chmod-tladmin multi-start multi-stop multi-restart sql-import update-2.10 update-2.11 update-2.12 update-2.13 update-2.14 all: dirs js skins robots data/config.pl -dirs: static/cv static/sf static/st data/log www +dirs: static/cv static/sf static/st data/log www www/feeds static/cv static/sf static/st: mkdir $@; for i in $$(seq -w 0 1 99); do mkdir "$@/$$i"; done -data/log www: +data/log www www/feeds: mkdir $@ @@ -152,3 +152,5 @@ update-2.13: all ${runpsql} < util/updates/update_2.13.sql $(multi-start) +update-2.14: all + @@ -13,10 +13,10 @@ Requirements global requirements: Linux, or an OS that resembles Linux. Chances are VNDB won't run on Windows. PostgreSQL 8.4+ - perl 5.10 recommended, 5.8 may also work + perl 5.12 recommended, 5.10 and 5.8 may also work A webserver that works with YAWF (lighttpd and Apache are known to work) - (perl 5.10 core modules are not listed.) + (perl 5.12 core modules are not listed.) util/vndb.pl: Algorithm::Diff::Fast @@ -35,6 +35,8 @@ Requirements DBD::Pg POE POE::Component::Pg (get it from http://g.blicky.net/poco-pg.git/) + Feed: + XML::Writer IRC: POE::Component::IRC URI::Escape diff --git a/data/global.pl b/data/global.pl index 0e7b6d8c..cc251851 100644 --- a/data/global.pl +++ b/data/global.pl @@ -109,6 +109,7 @@ our %M = ( log_dir => $ROOT.'/data/log', modules => { #API => {}, # disabled by default, not really needed + Feed => {}, RG => {}, Image => {}, #Anime => {}, # disabled by default, requires AniDB username/pass diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm new file mode 100644 index 00000000..4c157b2d --- /dev/null +++ b/lib/Multi/Feed.pm @@ -0,0 +1,185 @@ + +# +# Multi::Feed - Generates and updates Atom feeds +# + +package Multi::Feed; + +use strict; +use warnings; +use POE; +use XML::Writer; +use POSIX 'strftime'; +use Time::HiRes 'time'; +use VNDBUtil 'bb2html'; + + +sub spawn { + my $p = shift; + POE::Session->create( + package_states => [ + $p => [qw| _start shutdown generate write_atom log_stats |], + ], + heap => { + regenerate_interval => 900, # 15 min. + stats_interval => 86400, # daily + num_announcements => 10, + num_changes => 25, + num_posts => 25, + debug => 0, + @_, + stats => {}, # key = feed, value = [ count, total, max ] + }, + ); +} + + +sub _start { + $_[KERNEL]->alias_set('feed'); + $_[KERNEL]->yield('generate'); + $_[KERNEL]->alarm(log_stats => int((time+3)/$_[HEAP]{stats_interval}+1)*$_[HEAP]{stats_interval}); + $_[KERNEL]->sig(shutdown => 'shutdown'); +} + + +sub shutdown { + $_[KERNEL]->delay('generate'); + $_[KERNEL]->delay('log_stats'); + $_[KERNEL]->alias_remove('feed'); +} + + +sub generate { + $_[KERNEL]->alarm(generate => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval}); + + # announcements + $_[KERNEL]->post(pg => query => q{ + SELECT '/t'||t.id AS id, t.title, extract('epoch' from tp.date) AS published, + extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary + FROM threads t + JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1 + JOIN threads_boards tb ON tb.tid = t.id AND tb.type = 'an' + JOIN users u ON u.id = tp.uid + WHERE NOT t.hidden + ORDER BY t.id DESC + LIMIT ?}, [ $_[HEAP]{num_announcements} ], 'write_atom', + { + feed => 'announcements', + title => 'VNDB.org Site Announcements', + id => '/t/an', + }); + + # changes + $_[KERNEL]->post(pg => query => q{ + SELECT '/'||c.type||COALESCE(vr.vid, rr.rid, pr.pid)||'.'||c.rev AS id, + COALESCE(vr.title, rr.title, pr.name) AS title, extract('epoch' from c.added) AS updated, + u.username, u.id AS uid, c.comments AS summary + FROM changes c + LEFT JOIN vn_rev vr ON c.type = 'v' AND c.id = vr.id + LEFT JOIN releases_rev rr ON c.type = 'r' AND c.id = rr.id + LEFT JOIN producers_rev pr ON c.type = 'p' AND c.id = pr.id + JOIN users u ON u.id = c.requester + WHERE c.requester <> 1 + ORDER BY c.id DESC + LIMIT ?}, [ $_[HEAP]{num_changes} ], 'write_atom', + { + feed => 'changes', + title => 'VNDB.org Recent Changes', + id => '/hist', + }); + + # posts (this query isn't all that fast) + $_[KERNEL]->post(pg => query => q{ + SELECT '/t'||t.id||'.'||tp.num AS id, t.title||' (#'||tp.num||')' AS title, extract('epoch' from tp.date) AS published, + extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary + FROM threads_posts tp + JOIN threads t ON t.id = tp.tid + JOIN users u ON u.id = tp.uid + WHERE NOT tp.hidden AND NOT t.hidden + ORDER BY tp.date DESC + LIMIT ?}, [ $_[HEAP]{num_posts} ], 'write_atom', + { + feed => 'posts', + title => 'VNDB.org Recent Posts', + id => '/t', + }); +} + + +sub write_atom { # num, res, nfo, time + my $r = $_[ARG1]; + my $nfo = $_[ARG2]; + + my $start = time; + + my $updated = 0; + for(@$r) { + $updated = $_->{published} if $_->{published} && $_->{published} > $updated; + $updated = $_->{updated} if $_->{updated} && $_->{updated} > $updated; + } + + my $data; + my $x = XML::Writer->new(OUTPUT => \$data, DATA_MODE => 1, DATA_INDENT => 2); + $x->xmlDecl('UTF-8'); + $x->startTag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => $VNDB::S{url}.'/'); + $x->dataElement(title => $nfo->{title}); + $x->dataElement(updated => datetime($updated)); + $x->dataElement(id => $VNDB::S{url}.$nfo->{id}); + $x->emptyTag(link => rel => 'self', type => 'application/atom+xml', href => "$VNDB::S{url}/feeds/$nfo->{feed}.atom"); + $x->emptyTag(link => rel => 'alternate', type => 'text/html', href => $nfo->{id}); + + for(@$r) { + $x->startTag('entry'); + $x->dataElement(id => $VNDB::S{url}.$_->{id}); + $x->dataElement(title => $_->{title}); + $x->dataElement(updated => $_->{updated}?datetime($_->{updated}):datetime($_->{published})); + $x->dataElement(published => datetime($_->{published})) if $_->{published}; + if($_->{username}) { + $x->startTag('author'); + $x->dataElement(name => $_->{username}); + $x->dataElement(uri => '/u'.$_->{uid}) if $_->{uid}; + $x->endTag('author'); + } + $x->emptyTag(link => rel => 'alternate', type => 'text/html', href => $_->{id}); + $x->dataElement(summary => bb2html($_->{summary}, 200), type => 'html') if $_->{summary}; + $x->endTag('entry'); + } + + $x->endTag('feed'); + + open my $f, '>:utf8', "$VNDB::ROOT/www/feeds/$nfo->{feed}.atom" || die $!; + print $f $data; + close $f; + + $_[HEAP]{debug} && $_[KERNEL]->call(core => log => 'Wrote %s.atom (%d entries, sql:%4dms, perl:%4dms)', + $nfo->{feed}, scalar(@$r), $_[ARG3]*1000, (time-$start)*1000); + + $_[HEAP]{stats}{$nfo->{feed}} = [ 0, 0, 0 ] if !$_[HEAP]{stats}{$nfo->{feed}}; + my $time = ((time-$start)+$_[ARG3])*1000; + $_[HEAP]{stats}{$nfo->{feed}}[0]++; + $_[HEAP]{stats}{$nfo->{feed}}[1] += $time; + $_[HEAP]{stats}{$nfo->{feed}}[2] = $time if $_[HEAP]{stats}{$nfo->{feed}}[2] < $time; +} + + +sub log_stats { + $_[KERNEL]->alarm(log_stats => int((time+3)/$_[HEAP]{stats_interval}+1)*$_[HEAP]{stats_interval}); + + for (keys %{$_[HEAP]{stats}}) { + my $v = $_[HEAP]{stats}{$_}; + next if !$v->[0]; + $_[KERNEL]->call(core => log => 'Stats summary for %s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB', + $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s "$VNDB::ROOT/www/feeds/$_.atom")/1024); + } + $_[HEAP]{stats} = {}; +} + + +# non-POE helper function +sub datetime { + strftime('%Y-%m-%dT%H:%M:%SZ', gmtime shift); +} + + +1; + |