Feeds: Move feed generation form Multi to VNWeb

This means the feeds are now generated on demand rather than every 15
minutes. Main reason for originally implementing this into Multi was
because RSS feeds tended to get requested a *lot* and I didn't want
those requests to impact site performance, but now that RSS is almost
dead it doesn't really matter that much anymore. A caching layer can
still be added anyway.

Be sure to restart Multi and delete www/feeds/ after this change.

9 files changed, 81 insertions, 163 deletions

diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm
deleted file mode 100644
index ad52bace..00000000
--- a/lib/Multi/Feed.pm
+++ /dev/null
@@ -1,155 +0,0 @@
-
-#
-#  Multi::Feed  -  Generates and updates Atom feeds
-#
-
-package Multi::Feed;
-
-use strict;
-use warnings;
-use TUWF::XML;
-use Multi::Core;
-use POSIX 'strftime';
-use VNDB::BBCode;
-use VNDB::Config;
-
-my %stats; # key = feed, value = [ count, total, max ]
-
-
-sub run {
-  my $p = shift;
-  my %o = (
-    regenerate_interval => 600, # 10 min.
-    stats_interval => 86400, # daily
-    @_
-  );
-  push_watcher schedule 0, $o{regenerate_interval}, \&generate;
-  push_watcher schedule 0, $o{stats_interval}, \&stats;
-}
-
-
-sub generate {
-  # announcements
-  pg_cmd q{
-      SELECT '/'||t.id AS id, t.title, extract('epoch' from tp.date) AS published,
-         extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
-       FROM threads t
-       JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1
-       JOIN threads_boards tb ON tb.tid = t.id AND tb.type = 'an'
-       LEFT JOIN users u ON u.id = tp.uid
-      WHERE NOT t.hidden AND NOT t.private
-      ORDER BY t.id DESC
-      LIMIT $1},
-    [10],
-    sub { write_atom(announcements => '/t/an', 'VNDB Site Announcements', @_) };
-
-  # changes
-  pg_cmd q{
-      SELECT '/'||c.type||COALESCE(v.id, r.id, p.id, ca.id, s.id, d.id)||'.'||c.rev AS id,
-         COALESCE(v.title, r.title, p.name, ca.name, sa.name, d.title) AS title, extract('epoch' from c.added) AS updated,
-         u.username, u.id AS uid, c.comments AS summary
-      FROM changes c
-       LEFT JOIN vn v ON c.type = 'v' AND c.itemid = v.id
-       LEFT JOIN releases r ON c.type = 'r' AND c.itemid = r.id
-       LEFT JOIN producers p ON c.type = 'p' AND c.itemid = p.id
-       LEFT JOIN chars ca ON c.type = 'c' AND c.itemid = ca.id
-       LEFT JOIN docs d ON c.type = 'd' AND c.itemid = d.id
-       LEFT JOIN staff s ON c.type = 's' AND c.itemid = s.id
-       LEFT JOIN staff_alias sa ON sa.id = s.id AND sa.aid = s.aid
-       JOIN users u ON u.id = c.requester
-      WHERE c.requester <> 1
-      ORDER BY c.id DESC
-      LIMIT $1},
-    [25],
-    sub { write_atom(changes => '/hist', 'VNDB Recent Changes', @_); };
-
-  # posts
-  pg_cmd q{
-      SELECT '/'||t.id||'.'||tp.num AS id, t.title||' (#'||tp.num||')' AS title, extract('epoch' from tp.date) AS published,
-         extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
-       FROM threads_posts tp
-       JOIN threads t ON t.id = tp.tid
-       LEFT JOIN users u ON u.id = tp.uid
-      WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private
-      ORDER BY tp.date DESC
-      LIMIT $1},
-    [25],
-    sub { write_atom(posts => '/t', 'VNDB Recent Posts', @_); };
-}
-
-
-sub write_atom {
-  my($feed, $path, $title, $res, $sqltime) = @_;
-  return if pg_expect $res, 1;
-
-  my $start = AE::time;
-
-  my @r = $res->rowsAsHashes;
-  my $updated = 0;
-  for(@r) {
-    $updated = $_->{published} if $_->{published} && $_->{published} > $updated;
-    $updated = $_->{updated} if $_->{updated} && $_->{updated} > $updated;
-  }
-
-  my $data;
-  my $x = TUWF::XML->new(write => sub { $data .= shift }, pretty => 2);
-  $x->xml();
-  $x->tag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => config->{url}.'/');
-  $x->tag(title => $title);
-  $x->tag(updated => datetime($updated));
-  $x->tag(id => config->{url}.$path);
-  $x->tag(link => rel => 'self', type => 'application/atom+xml', href => config->{url}."/feeds/$feed.atom", undef);
-  $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$path, undef);
-
-  for(@r) {
-    $x->tag('entry');
-    $x->tag(id => config->{url}.$_->{id});
-    $x->tag(title => $_->{title});
-    $x->tag(updated => datetime($_->{updated} || $_->{published}));
-    $x->tag(published => datetime($_->{published})) if $_->{published};
-    if($_->{username}) {
-      $x->tag('author');
-      $x->tag(name => $_->{username});
-      $x->tag(uri => config->{url}.'/u'.$_->{uid}) if $_->{uid};
-      $x->end;
-    }
-    $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$_->{id}, undef);
-    $x->tag('summary', type => 'html', bb_format $_->{summary}) if $_->{summary};
-    $x->end('entry');
-  }
-
-  $x->end('feed');
-
-  open my $f, '>:utf8', config->{root}."/www/feeds/$feed.atom" || die $!;
-  print $f $data;
-  close $f;
-
-  AE::log debug => sprintf 'Wrote %16s.atom (%d entries, sql:%4dms, perl:%4dms)',
-    $feed, scalar(@r), $sqltime*1000, (AE::time-$start)*1000;
-
-  my $time = ((AE::time-$start)+$sqltime)*1000;
-  $stats{$feed} = [ 0, 0, 0 ] if !$stats{$feed};
-  $stats{$feed}[0]++;
-  $stats{$feed}[1] += $time;
-  $stats{$feed}[2] = $time if $stats{$feed}[2] < $time;
-}
-
-
-sub stats {
-  for (keys %stats) {
-    my $v = $stats{$_};
-    next if !$v->[0];
-    AE::log info => sprintf 'Stats summary for %16s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB',
-      $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s config->{root}."/www/feeds/$_.atom")/1024;
-  }
-  %stats = ();
-}
-
-
-sub datetime {
-  strftime('%Y-%m-%dT%H:%M:%SZ', gmtime shift);
-}
-
-
-1;
-
diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm
index 6a1d8adf..01b67a58 100644
--- a/lib/VNDB/Config.pm
+++ b/lib/VNDB/Config.pm
@@ -38,7 +38,6 @@ my $config = {
 
     Multi => {
         Core        => {},
-        Feed        => {},
         Maintenance => {},
     },
 };
diff --git a/lib/VNWeb/Filters.pm b/lib/VNWeb/Filters.pm
index 79aa93dd..75e82336 100644
--- a/lib/VNWeb/Filters.pm
+++ b/lib/VNWeb/Filters.pm
@@ -6,7 +6,6 @@ package VNWeb::Filters;
 # we'll need to support these filters for the forseeable future.
 
 use VNWeb::Prelude;
-use POSIX 'strftime';
 use Exporter 'import';
 
 our @EXPORT = qw/filter_parse filter_vn_query filter_release_query/;
diff --git a/lib/VNWeb/Misc/Feeds.pm b/lib/VNWeb/Misc/Feeds.pm
new file mode 100644
index 00000000..6d3a3eba
--- /dev/null
+++ b/lib/VNWeb/Misc/Feeds.pm
@@ -0,0 +1,78 @@
+package VNWeb::Misc::Feeds;
+
+use VNWeb::Prelude;
+use TUWF::XML ':xml';
+
+
+sub datetime { strftime '%Y-%m-%dT%H:%M:%SZ', gmtime shift }
+
+
+sub feed {
+    my($path, $title, $data) = @_;
+    my $base = tuwf->reqBaseURI();
+
+    xml;
+    tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => "$base/", sub {
+        tag title => $title;
+        tag updated => datetime max grep $_, map +($_->{published}, $_->{updated}), @$data;
+        tag id => $base.$path;
+        tag link => rel => 'self', type => 'application/atom+xml', href => $base.tuwf->reqPath(), undef;
+        tag link => rel => 'alternate', type => 'text/html', href => $base.$path, undef;
+
+        tag entry => sub {
+            tag id => "$base/$_->{id}";
+            tag title => $_->{title};
+            tag updated => datetime($_->{updated} || $_->{published});
+            tag published => datetime $_->{published} if $_->{published};
+            tag author => sub {
+                tag name => $_->{user_name};
+                tag uri => "$base/u$_->{user_id}";
+            } if $_->{user_id};
+            tag link => rel => 'alternate', type => 'text/html', href => "$base/$_->{id}", undef;
+            tag summary => type => 'html', bb_format $_->{summary}, maxlength => 300 if $_->{summary};
+        } for @$data;
+    }
+}
+
+
+TUWF::get qr{/feeds/announcements.atom}, sub {
+    feed '/t/an', 'VNDB Site Announcements', tuwf->dbAlli('
+      SELECT t.id, t.title, tp.msg AS summary
+           , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), '
+        FROM threads t
+        JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1
+        JOIN threads_boards tb ON tb.tid = t.id AND tb.type = \'an\'
+        LEFT JOIN users u ON u.id = tp.uid
+       WHERE NOT t.hidden AND NOT t.private
+       ORDER BY tb.tid DESC
+       LIMIT 10'
+    );
+};
+
+
+TUWF::get qr{/feeds/changes.atom}, sub {
+    my($lst) = VNWeb::Misc::History::fetch(undef, undef, {m=>1,h=>1,p=>1}, {results=>25});
+    for (@$lst) {
+        $_->{id}      = "$_->{type}$_->{itemid}.$_->{rev}";
+        $_->{summary} = $_->{comments};
+        $_->{updated} = $_->{added};
+    }
+    feed '/hist', 'VNDB Recent Changes', $lst;
+};
+
+
+TUWF::get qr{/feeds/posts.atom}, sub {
+    feed '/t', 'VNDB Recent Posts', tuwf->dbAlli('
+      SELECT t.id||\'.\'||tp.num AS id, t.title||\' (#\'||tp.num||\')\' AS title, tp.msg AS summary
+           , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), '
+        FROM threads_posts tp
+        JOIN threads t ON t.id = tp.tid
+        LEFT JOIN users u ON u.id = tp.uid
+       WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private
+       ORDER BY tp.date DESC
+       LIMIT ', \25
+    );
+};
+
+
+1;
diff --git a/lib/VNWeb/Misc/History.pm b/lib/VNWeb/Misc/History.pm
index 691d035a..dcddfb4b 100644
--- a/lib/VNWeb/Misc/History.pm
+++ b/lib/VNWeb/Misc/History.pm
@@ -3,7 +3,7 @@ package VNWeb::Misc::History;
 use VNWeb::Prelude;
 
 
-# Also used by Misc::HomePage
+# Also used by Misc::HomePage and Misc::Feeds
 sub fetch {
     my($type, $id, $filt, $opt) = @_;
 
diff --git a/lib/VNWeb/Misc/HomePage.pm b/lib/VNWeb/Misc/HomePage.pm
index 612cfa04..f71d776b 100644
--- a/lib/VNWeb/Misc/HomePage.pm
+++ b/lib/VNWeb/Misc/HomePage.pm
@@ -3,7 +3,6 @@ package VNWeb::Misc::HomePage;
 use VNWeb::Prelude;
 use VNWeb::Filters;
 use VNWeb::Discussions::Lib 'enrich_boards';
-use POSIX 'strftime';
 
 
 sub screens_ {
diff --git a/lib/VNWeb/Prelude.pm b/lib/VNWeb/Prelude.pm
index e202974c..198d09e2 100644
--- a/lib/VNWeb/Prelude.pm
+++ b/lib/VNWeb/Prelude.pm
@@ -8,7 +8,7 @@
 #  use Exporter 'import';
 #  use Time::HiRes 'time';
 #  use List::Util 'min', 'max', 'sum';
-#  use POSIX 'ceil', 'floor';
+#  use POSIX 'ceil', 'floor', 'strftime';
 #
 #  use VNDBUtil;
 #  use VNDB::BBCode;
@@ -53,7 +53,7 @@ sub import {
     use Exporter 'import';
     use Time::HiRes 'time';
     use List::Util 'min', 'max', 'sum';
-    use POSIX 'ceil', 'floor';
+    use POSIX 'ceil', 'floor', 'strftime';
 
     use VNDBUtil;
     use VNDB::BBCode;
diff --git a/lib/VNWeb/ULists/List.pm b/lib/VNWeb/ULists/List.pm
index 3d2bf681..dbdfddc9 100644
--- a/lib/VNWeb/ULists/List.pm
+++ b/lib/VNWeb/ULists/List.pm
@@ -3,7 +3,6 @@ package VNWeb::ULists::Main;
 use VNWeb::Prelude;
 use VNWeb::ULists::Lib;
 use VNWeb::Releases::Lib;
-use POSIX 'strftime';
 
 
 sub opt {
diff --git a/lib/VNWeb/VN/Page.pm b/lib/VNWeb/VN/Page.pm
index ae32ecf3..ed4f6e75 100644
--- a/lib/VNWeb/VN/Page.pm
+++ b/lib/VNWeb/VN/Page.pm
@@ -4,7 +4,6 @@ use VNWeb::Prelude;
 use VNWeb::Releases::Lib;
 use VNWeb::Images::Lib qw/image_flagging_display image_ enrich_image_obj/;
 use VNDB::Func 'fmtrating';
-use POSIX 'strftime';
 
 
 # Enrich everything necessary to at least render infobox_().