diff options
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | Makefile | 4 | ||||
-rw-r--r-- | lib/Multi/Feed.pm | 155 | ||||
-rw-r--r-- | lib/VNDB/Config.pm | 1 | ||||
-rw-r--r-- | lib/VNWeb/Filters.pm | 1 | ||||
-rw-r--r-- | lib/VNWeb/Misc/Feeds.pm | 78 | ||||
-rw-r--r-- | lib/VNWeb/Misc/History.pm | 2 | ||||
-rw-r--r-- | lib/VNWeb/Misc/HomePage.pm | 1 | ||||
-rw-r--r-- | lib/VNWeb/Prelude.pm | 4 | ||||
-rw-r--r-- | lib/VNWeb/ULists/List.pm | 1 | ||||
-rw-r--r-- | lib/VNWeb/VN/Page.pm | 1 |
11 files changed, 83 insertions, 166 deletions
@@ -18,7 +18,6 @@ /static/f/plain.js /static/f/plain.min.js /static/f/plain.min.js.gz -/static/feeds/ /static/s/*/style.css /static/s/*/style.min.css /static/s/*/style.min.css.gz @@ -24,7 +24,7 @@ ALL_KEEP=\ static/ch static/cv static/sf static/st \ - data/log static/f www www/feeds www/api \ + data/log static/f www www/api \ data/conf.pl \ www/robots.txt static/robots.txt @@ -64,7 +64,7 @@ static/ch static/cv static/sf static/st: mkdir -p $@; for i in $$(seq -w 0 1 99); do mkdir -p "$@/$$i"; done -data/log www www/feeds www/api static/f: +data/log www www/api static/f: mkdir -p $@ data/conf.pl: diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm deleted file mode 100644 index ad52bace..00000000 --- a/lib/Multi/Feed.pm +++ /dev/null @@ -1,155 +0,0 @@ - -# -# Multi::Feed - Generates and updates Atom feeds -# - -package Multi::Feed; - -use strict; -use warnings; -use TUWF::XML; -use Multi::Core; -use POSIX 'strftime'; -use VNDB::BBCode; -use VNDB::Config; - -my %stats; # key = feed, value = [ count, total, max ] - - -sub run { - my $p = shift; - my %o = ( - regenerate_interval => 600, # 10 min. - stats_interval => 86400, # daily - @_ - ); - push_watcher schedule 0, $o{regenerate_interval}, \&generate; - push_watcher schedule 0, $o{stats_interval}, \&stats; -} - - -sub generate { - # announcements - pg_cmd q{ - SELECT '/'||t.id AS id, t.title, extract('epoch' from tp.date) AS published, - extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary - FROM threads t - JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1 - JOIN threads_boards tb ON tb.tid = t.id AND tb.type = 'an' - LEFT JOIN users u ON u.id = tp.uid - WHERE NOT t.hidden AND NOT t.private - ORDER BY t.id DESC - LIMIT $1}, - [10], - sub { write_atom(announcements => '/t/an', 'VNDB Site Announcements', @_) }; - - # changes - pg_cmd q{ - SELECT '/'||c.type||COALESCE(v.id, r.id, p.id, ca.id, s.id, d.id)||'.'||c.rev AS id, - COALESCE(v.title, r.title, p.name, ca.name, sa.name, d.title) AS title, extract('epoch' from c.added) AS updated, - u.username, u.id AS uid, c.comments AS summary - FROM changes c - LEFT JOIN vn v ON c.type = 'v' AND c.itemid = v.id - LEFT JOIN releases r ON c.type = 'r' AND c.itemid = r.id - LEFT JOIN producers p ON c.type = 'p' AND c.itemid = p.id - LEFT JOIN chars ca ON c.type = 'c' AND c.itemid = ca.id - LEFT JOIN docs d ON c.type = 'd' AND c.itemid = d.id - LEFT JOIN staff s ON c.type = 's' AND c.itemid = s.id - LEFT JOIN staff_alias sa ON sa.id = s.id AND sa.aid = s.aid - JOIN users u ON u.id = c.requester - WHERE c.requester <> 1 - ORDER BY c.id DESC - LIMIT $1}, - [25], - sub { write_atom(changes => '/hist', 'VNDB Recent Changes', @_); }; - - # posts - pg_cmd q{ - SELECT '/'||t.id||'.'||tp.num AS id, t.title||' (#'||tp.num||')' AS title, extract('epoch' from tp.date) AS published, - extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary - FROM threads_posts tp - JOIN threads t ON t.id = tp.tid - LEFT JOIN users u ON u.id = tp.uid - WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private - ORDER BY tp.date DESC - LIMIT $1}, - [25], - sub { write_atom(posts => '/t', 'VNDB Recent Posts', @_); }; -} - - -sub write_atom { - my($feed, $path, $title, $res, $sqltime) = @_; - return if pg_expect $res, 1; - - my $start = AE::time; - - my @r = $res->rowsAsHashes; - my $updated = 0; - for(@r) { - $updated = $_->{published} if $_->{published} && $_->{published} > $updated; - $updated = $_->{updated} if $_->{updated} && $_->{updated} > $updated; - } - - my $data; - my $x = TUWF::XML->new(write => sub { $data .= shift }, pretty => 2); - $x->xml(); - $x->tag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => config->{url}.'/'); - $x->tag(title => $title); - $x->tag(updated => datetime($updated)); - $x->tag(id => config->{url}.$path); - $x->tag(link => rel => 'self', type => 'application/atom+xml', href => config->{url}."/feeds/$feed.atom", undef); - $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$path, undef); - - for(@r) { - $x->tag('entry'); - $x->tag(id => config->{url}.$_->{id}); - $x->tag(title => $_->{title}); - $x->tag(updated => datetime($_->{updated} || $_->{published})); - $x->tag(published => datetime($_->{published})) if $_->{published}; - if($_->{username}) { - $x->tag('author'); - $x->tag(name => $_->{username}); - $x->tag(uri => config->{url}.'/u'.$_->{uid}) if $_->{uid}; - $x->end; - } - $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$_->{id}, undef); - $x->tag('summary', type => 'html', bb_format $_->{summary}) if $_->{summary}; - $x->end('entry'); - } - - $x->end('feed'); - - open my $f, '>:utf8', config->{root}."/www/feeds/$feed.atom" || die $!; - print $f $data; - close $f; - - AE::log debug => sprintf 'Wrote %16s.atom (%d entries, sql:%4dms, perl:%4dms)', - $feed, scalar(@r), $sqltime*1000, (AE::time-$start)*1000; - - my $time = ((AE::time-$start)+$sqltime)*1000; - $stats{$feed} = [ 0, 0, 0 ] if !$stats{$feed}; - $stats{$feed}[0]++; - $stats{$feed}[1] += $time; - $stats{$feed}[2] = $time if $stats{$feed}[2] < $time; -} - - -sub stats { - for (keys %stats) { - my $v = $stats{$_}; - next if !$v->[0]; - AE::log info => sprintf 'Stats summary for %16s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB', - $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s config->{root}."/www/feeds/$_.atom")/1024; - } - %stats = (); -} - - -sub datetime { - strftime('%Y-%m-%dT%H:%M:%SZ', gmtime shift); -} - - -1; - diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm index 6a1d8adf..01b67a58 100644 --- a/lib/VNDB/Config.pm +++ b/lib/VNDB/Config.pm @@ -38,7 +38,6 @@ my $config = { Multi => { Core => {}, - Feed => {}, Maintenance => {}, }, }; diff --git a/lib/VNWeb/Filters.pm b/lib/VNWeb/Filters.pm index 79aa93dd..75e82336 100644 --- a/lib/VNWeb/Filters.pm +++ b/lib/VNWeb/Filters.pm @@ -6,7 +6,6 @@ package VNWeb::Filters; # we'll need to support these filters for the forseeable future. use VNWeb::Prelude; -use POSIX 'strftime'; use Exporter 'import'; our @EXPORT = qw/filter_parse filter_vn_query filter_release_query/; diff --git a/lib/VNWeb/Misc/Feeds.pm b/lib/VNWeb/Misc/Feeds.pm new file mode 100644 index 00000000..6d3a3eba --- /dev/null +++ b/lib/VNWeb/Misc/Feeds.pm @@ -0,0 +1,78 @@ +package VNWeb::Misc::Feeds; + +use VNWeb::Prelude; +use TUWF::XML ':xml'; + + +sub datetime { strftime '%Y-%m-%dT%H:%M:%SZ', gmtime shift } + + +sub feed { + my($path, $title, $data) = @_; + my $base = tuwf->reqBaseURI(); + + xml; + tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => "$base/", sub { + tag title => $title; + tag updated => datetime max grep $_, map +($_->{published}, $_->{updated}), @$data; + tag id => $base.$path; + tag link => rel => 'self', type => 'application/atom+xml', href => $base.tuwf->reqPath(), undef; + tag link => rel => 'alternate', type => 'text/html', href => $base.$path, undef; + + tag entry => sub { + tag id => "$base/$_->{id}"; + tag title => $_->{title}; + tag updated => datetime($_->{updated} || $_->{published}); + tag published => datetime $_->{published} if $_->{published}; + tag author => sub { + tag name => $_->{user_name}; + tag uri => "$base/u$_->{user_id}"; + } if $_->{user_id}; + tag link => rel => 'alternate', type => 'text/html', href => "$base/$_->{id}", undef; + tag summary => type => 'html', bb_format $_->{summary}, maxlength => 300 if $_->{summary}; + } for @$data; + } +} + + +TUWF::get qr{/feeds/announcements.atom}, sub { + feed '/t/an', 'VNDB Site Announcements', tuwf->dbAlli(' + SELECT t.id, t.title, tp.msg AS summary + , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), ' + FROM threads t + JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1 + JOIN threads_boards tb ON tb.tid = t.id AND tb.type = \'an\' + LEFT JOIN users u ON u.id = tp.uid + WHERE NOT t.hidden AND NOT t.private + ORDER BY tb.tid DESC + LIMIT 10' + ); +}; + + +TUWF::get qr{/feeds/changes.atom}, sub { + my($lst) = VNWeb::Misc::History::fetch(undef, undef, {m=>1,h=>1,p=>1}, {results=>25}); + for (@$lst) { + $_->{id} = "$_->{type}$_->{itemid}.$_->{rev}"; + $_->{summary} = $_->{comments}; + $_->{updated} = $_->{added}; + } + feed '/hist', 'VNDB Recent Changes', $lst; +}; + + +TUWF::get qr{/feeds/posts.atom}, sub { + feed '/t', 'VNDB Recent Posts', tuwf->dbAlli(' + SELECT t.id||\'.\'||tp.num AS id, t.title||\' (#\'||tp.num||\')\' AS title, tp.msg AS summary + , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), ' + FROM threads_posts tp + JOIN threads t ON t.id = tp.tid + LEFT JOIN users u ON u.id = tp.uid + WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private + ORDER BY tp.date DESC + LIMIT ', \25 + ); +}; + + +1; diff --git a/lib/VNWeb/Misc/History.pm b/lib/VNWeb/Misc/History.pm index 691d035a..dcddfb4b 100644 --- a/lib/VNWeb/Misc/History.pm +++ b/lib/VNWeb/Misc/History.pm @@ -3,7 +3,7 @@ package VNWeb::Misc::History; use VNWeb::Prelude; -# Also used by Misc::HomePage +# Also used by Misc::HomePage and Misc::Feeds sub fetch { my($type, $id, $filt, $opt) = @_; diff --git a/lib/VNWeb/Misc/HomePage.pm b/lib/VNWeb/Misc/HomePage.pm index 612cfa04..f71d776b 100644 --- a/lib/VNWeb/Misc/HomePage.pm +++ b/lib/VNWeb/Misc/HomePage.pm @@ -3,7 +3,6 @@ package VNWeb::Misc::HomePage; use VNWeb::Prelude; use VNWeb::Filters; use VNWeb::Discussions::Lib 'enrich_boards'; -use POSIX 'strftime'; sub screens_ { diff --git a/lib/VNWeb/Prelude.pm b/lib/VNWeb/Prelude.pm index e202974c..198d09e2 100644 --- a/lib/VNWeb/Prelude.pm +++ b/lib/VNWeb/Prelude.pm @@ -8,7 +8,7 @@ # use Exporter 'import'; # use Time::HiRes 'time'; # use List::Util 'min', 'max', 'sum'; -# use POSIX 'ceil', 'floor'; +# use POSIX 'ceil', 'floor', 'strftime'; # # use VNDBUtil; # use VNDB::BBCode; @@ -53,7 +53,7 @@ sub import { use Exporter 'import'; use Time::HiRes 'time'; use List::Util 'min', 'max', 'sum'; - use POSIX 'ceil', 'floor'; + use POSIX 'ceil', 'floor', 'strftime'; use VNDBUtil; use VNDB::BBCode; diff --git a/lib/VNWeb/ULists/List.pm b/lib/VNWeb/ULists/List.pm index 3d2bf681..dbdfddc9 100644 --- a/lib/VNWeb/ULists/List.pm +++ b/lib/VNWeb/ULists/List.pm @@ -3,7 +3,6 @@ package VNWeb::ULists::Main; use VNWeb::Prelude; use VNWeb::ULists::Lib; use VNWeb::Releases::Lib; -use POSIX 'strftime'; sub opt { diff --git a/lib/VNWeb/VN/Page.pm b/lib/VNWeb/VN/Page.pm index ae32ecf3..ed4f6e75 100644 --- a/lib/VNWeb/VN/Page.pm +++ b/lib/VNWeb/VN/Page.pm @@ -4,7 +4,6 @@ use VNWeb::Prelude; use VNWeb::Releases::Lib; use VNWeb::Images::Lib qw/image_flagging_display image_ enrich_image_obj/; use VNDB::Func 'fmtrating'; -use POSIX 'strftime'; # Enrich everything necessary to at least render infobox_(). |