summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2020-09-17 11:38:16 +0200
committerYorhel <git@yorhel.nl>2020-09-17 11:41:43 +0200
commit266d9e9b9f4fd3f840f950aefe7e82ecbc0f1689 (patch)
tree960b83064d2ae5aada9c091dde2c13c5526a82b8
parent62f6803b581d174db4f7259c7dcfcca19730fc89 (diff)
Feeds: Move feed generation form Multi to VNWeb
This means the feeds are now generated on demand rather than every 15 minutes. Main reason for originally implementing this into Multi was because RSS feeds tended to get requested a *lot* and I didn't want those requests to impact site performance, but now that RSS is almost dead it doesn't really matter that much anymore. A caching layer can still be added anyway. Be sure to restart Multi and delete www/feeds/ after this change.
-rw-r--r--.gitignore1
-rw-r--r--Makefile4
-rw-r--r--lib/Multi/Feed.pm155
-rw-r--r--lib/VNDB/Config.pm1
-rw-r--r--lib/VNWeb/Filters.pm1
-rw-r--r--lib/VNWeb/Misc/Feeds.pm78
-rw-r--r--lib/VNWeb/Misc/History.pm2
-rw-r--r--lib/VNWeb/Misc/HomePage.pm1
-rw-r--r--lib/VNWeb/Prelude.pm4
-rw-r--r--lib/VNWeb/ULists/List.pm1
-rw-r--r--lib/VNWeb/VN/Page.pm1
11 files changed, 83 insertions, 166 deletions
diff --git a/.gitignore b/.gitignore
index 584a6483..b93194e2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,6 @@
/static/f/plain.js
/static/f/plain.min.js
/static/f/plain.min.js.gz
-/static/feeds/
/static/s/*/style.css
/static/s/*/style.min.css
/static/s/*/style.min.css.gz
diff --git a/Makefile b/Makefile
index 35151029..9f87f2f8 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@
ALL_KEEP=\
static/ch static/cv static/sf static/st \
- data/log static/f www www/feeds www/api \
+ data/log static/f www www/api \
data/conf.pl \
www/robots.txt static/robots.txt
@@ -64,7 +64,7 @@ static/ch static/cv static/sf static/st:
mkdir -p $@;
for i in $$(seq -w 0 1 99); do mkdir -p "$@/$$i"; done
-data/log www www/feeds www/api static/f:
+data/log www www/api static/f:
mkdir -p $@
data/conf.pl:
diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm
deleted file mode 100644
index ad52bace..00000000
--- a/lib/Multi/Feed.pm
+++ /dev/null
@@ -1,155 +0,0 @@
-
-#
-# Multi::Feed - Generates and updates Atom feeds
-#
-
-package Multi::Feed;
-
-use strict;
-use warnings;
-use TUWF::XML;
-use Multi::Core;
-use POSIX 'strftime';
-use VNDB::BBCode;
-use VNDB::Config;
-
-my %stats; # key = feed, value = [ count, total, max ]
-
-
-sub run {
- my $p = shift;
- my %o = (
- regenerate_interval => 600, # 10 min.
- stats_interval => 86400, # daily
- @_
- );
- push_watcher schedule 0, $o{regenerate_interval}, \&generate;
- push_watcher schedule 0, $o{stats_interval}, \&stats;
-}
-
-
-sub generate {
- # announcements
- pg_cmd q{
- SELECT '/'||t.id AS id, t.title, extract('epoch' from tp.date) AS published,
- extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
- FROM threads t
- JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1
- JOIN threads_boards tb ON tb.tid = t.id AND tb.type = 'an'
- LEFT JOIN users u ON u.id = tp.uid
- WHERE NOT t.hidden AND NOT t.private
- ORDER BY t.id DESC
- LIMIT $1},
- [10],
- sub { write_atom(announcements => '/t/an', 'VNDB Site Announcements', @_) };
-
- # changes
- pg_cmd q{
- SELECT '/'||c.type||COALESCE(v.id, r.id, p.id, ca.id, s.id, d.id)||'.'||c.rev AS id,
- COALESCE(v.title, r.title, p.name, ca.name, sa.name, d.title) AS title, extract('epoch' from c.added) AS updated,
- u.username, u.id AS uid, c.comments AS summary
- FROM changes c
- LEFT JOIN vn v ON c.type = 'v' AND c.itemid = v.id
- LEFT JOIN releases r ON c.type = 'r' AND c.itemid = r.id
- LEFT JOIN producers p ON c.type = 'p' AND c.itemid = p.id
- LEFT JOIN chars ca ON c.type = 'c' AND c.itemid = ca.id
- LEFT JOIN docs d ON c.type = 'd' AND c.itemid = d.id
- LEFT JOIN staff s ON c.type = 's' AND c.itemid = s.id
- LEFT JOIN staff_alias sa ON sa.id = s.id AND sa.aid = s.aid
- JOIN users u ON u.id = c.requester
- WHERE c.requester <> 1
- ORDER BY c.id DESC
- LIMIT $1},
- [25],
- sub { write_atom(changes => '/hist', 'VNDB Recent Changes', @_); };
-
- # posts
- pg_cmd q{
- SELECT '/'||t.id||'.'||tp.num AS id, t.title||' (#'||tp.num||')' AS title, extract('epoch' from tp.date) AS published,
- extract('epoch' from tp.edited) AS updated, u.username, u.id AS uid, tp.msg AS summary
- FROM threads_posts tp
- JOIN threads t ON t.id = tp.tid
- LEFT JOIN users u ON u.id = tp.uid
- WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private
- ORDER BY tp.date DESC
- LIMIT $1},
- [25],
- sub { write_atom(posts => '/t', 'VNDB Recent Posts', @_); };
-}
-
-
-sub write_atom {
- my($feed, $path, $title, $res, $sqltime) = @_;
- return if pg_expect $res, 1;
-
- my $start = AE::time;
-
- my @r = $res->rowsAsHashes;
- my $updated = 0;
- for(@r) {
- $updated = $_->{published} if $_->{published} && $_->{published} > $updated;
- $updated = $_->{updated} if $_->{updated} && $_->{updated} > $updated;
- }
-
- my $data;
- my $x = TUWF::XML->new(write => sub { $data .= shift }, pretty => 2);
- $x->xml();
- $x->tag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => config->{url}.'/');
- $x->tag(title => $title);
- $x->tag(updated => datetime($updated));
- $x->tag(id => config->{url}.$path);
- $x->tag(link => rel => 'self', type => 'application/atom+xml', href => config->{url}."/feeds/$feed.atom", undef);
- $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$path, undef);
-
- for(@r) {
- $x->tag('entry');
- $x->tag(id => config->{url}.$_->{id});
- $x->tag(title => $_->{title});
- $x->tag(updated => datetime($_->{updated} || $_->{published}));
- $x->tag(published => datetime($_->{published})) if $_->{published};
- if($_->{username}) {
- $x->tag('author');
- $x->tag(name => $_->{username});
- $x->tag(uri => config->{url}.'/u'.$_->{uid}) if $_->{uid};
- $x->end;
- }
- $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$_->{id}, undef);
- $x->tag('summary', type => 'html', bb_format $_->{summary}) if $_->{summary};
- $x->end('entry');
- }
-
- $x->end('feed');
-
- open my $f, '>:utf8', config->{root}."/www/feeds/$feed.atom" || die $!;
- print $f $data;
- close $f;
-
- AE::log debug => sprintf 'Wrote %16s.atom (%d entries, sql:%4dms, perl:%4dms)',
- $feed, scalar(@r), $sqltime*1000, (AE::time-$start)*1000;
-
- my $time = ((AE::time-$start)+$sqltime)*1000;
- $stats{$feed} = [ 0, 0, 0 ] if !$stats{$feed};
- $stats{$feed}[0]++;
- $stats{$feed}[1] += $time;
- $stats{$feed}[2] = $time if $stats{$feed}[2] < $time;
-}
-
-
-sub stats {
- for (keys %stats) {
- my $v = $stats{$_};
- next if !$v->[0];
- AE::log info => sprintf 'Stats summary for %16s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB',
- $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s config->{root}."/www/feeds/$_.atom")/1024;
- }
- %stats = ();
-}
-
-
-sub datetime {
- strftime('%Y-%m-%dT%H:%M:%SZ', gmtime shift);
-}
-
-
-1;
-
diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm
index 6a1d8adf..01b67a58 100644
--- a/lib/VNDB/Config.pm
+++ b/lib/VNDB/Config.pm
@@ -38,7 +38,6 @@ my $config = {
Multi => {
Core => {},
- Feed => {},
Maintenance => {},
},
};
diff --git a/lib/VNWeb/Filters.pm b/lib/VNWeb/Filters.pm
index 79aa93dd..75e82336 100644
--- a/lib/VNWeb/Filters.pm
+++ b/lib/VNWeb/Filters.pm
@@ -6,7 +6,6 @@ package VNWeb::Filters;
# we'll need to support these filters for the forseeable future.
use VNWeb::Prelude;
-use POSIX 'strftime';
use Exporter 'import';
our @EXPORT = qw/filter_parse filter_vn_query filter_release_query/;
diff --git a/lib/VNWeb/Misc/Feeds.pm b/lib/VNWeb/Misc/Feeds.pm
new file mode 100644
index 00000000..6d3a3eba
--- /dev/null
+++ b/lib/VNWeb/Misc/Feeds.pm
@@ -0,0 +1,78 @@
+package VNWeb::Misc::Feeds;
+
+use VNWeb::Prelude;
+use TUWF::XML ':xml';
+
+
+sub datetime { strftime '%Y-%m-%dT%H:%M:%SZ', gmtime shift }
+
+
+sub feed {
+ my($path, $title, $data) = @_;
+ my $base = tuwf->reqBaseURI();
+
+ xml;
+ tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => "$base/", sub {
+ tag title => $title;
+ tag updated => datetime max grep $_, map +($_->{published}, $_->{updated}), @$data;
+ tag id => $base.$path;
+ tag link => rel => 'self', type => 'application/atom+xml', href => $base.tuwf->reqPath(), undef;
+ tag link => rel => 'alternate', type => 'text/html', href => $base.$path, undef;
+
+ tag entry => sub {
+ tag id => "$base/$_->{id}";
+ tag title => $_->{title};
+ tag updated => datetime($_->{updated} || $_->{published});
+ tag published => datetime $_->{published} if $_->{published};
+ tag author => sub {
+ tag name => $_->{user_name};
+ tag uri => "$base/u$_->{user_id}";
+ } if $_->{user_id};
+ tag link => rel => 'alternate', type => 'text/html', href => "$base/$_->{id}", undef;
+ tag summary => type => 'html', bb_format $_->{summary}, maxlength => 300 if $_->{summary};
+ } for @$data;
+ }
+}
+
+
+TUWF::get qr{/feeds/announcements.atom}, sub {
+ feed '/t/an', 'VNDB Site Announcements', tuwf->dbAlli('
+ SELECT t.id, t.title, tp.msg AS summary
+ , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), '
+ FROM threads t
+ JOIN threads_posts tp ON tp.tid = t.id AND tp.num = 1
+ JOIN threads_boards tb ON tb.tid = t.id AND tb.type = \'an\'
+ LEFT JOIN users u ON u.id = tp.uid
+ WHERE NOT t.hidden AND NOT t.private
+ ORDER BY tb.tid DESC
+ LIMIT 10'
+ );
+};
+
+
+TUWF::get qr{/feeds/changes.atom}, sub {
+ my($lst) = VNWeb::Misc::History::fetch(undef, undef, {m=>1,h=>1,p=>1}, {results=>25});
+ for (@$lst) {
+ $_->{id} = "$_->{type}$_->{itemid}.$_->{rev}";
+ $_->{summary} = $_->{comments};
+ $_->{updated} = $_->{added};
+ }
+ feed '/hist', 'VNDB Recent Changes', $lst;
+};
+
+
+TUWF::get qr{/feeds/posts.atom}, sub {
+ feed '/t', 'VNDB Recent Posts', tuwf->dbAlli('
+ SELECT t.id||\'.\'||tp.num AS id, t.title||\' (#\'||tp.num||\')\' AS title, tp.msg AS summary
+ , ', sql_totime('tp.date'), 'AS published,', sql_totime('tp.edited'), 'AS updated,', sql_user(), '
+ FROM threads_posts tp
+ JOIN threads t ON t.id = tp.tid
+ LEFT JOIN users u ON u.id = tp.uid
+ WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private
+ ORDER BY tp.date DESC
+ LIMIT ', \25
+ );
+};
+
+
+1;
diff --git a/lib/VNWeb/Misc/History.pm b/lib/VNWeb/Misc/History.pm
index 691d035a..dcddfb4b 100644
--- a/lib/VNWeb/Misc/History.pm
+++ b/lib/VNWeb/Misc/History.pm
@@ -3,7 +3,7 @@ package VNWeb::Misc::History;
use VNWeb::Prelude;
-# Also used by Misc::HomePage
+# Also used by Misc::HomePage and Misc::Feeds
sub fetch {
my($type, $id, $filt, $opt) = @_;
diff --git a/lib/VNWeb/Misc/HomePage.pm b/lib/VNWeb/Misc/HomePage.pm
index 612cfa04..f71d776b 100644
--- a/lib/VNWeb/Misc/HomePage.pm
+++ b/lib/VNWeb/Misc/HomePage.pm
@@ -3,7 +3,6 @@ package VNWeb::Misc::HomePage;
use VNWeb::Prelude;
use VNWeb::Filters;
use VNWeb::Discussions::Lib 'enrich_boards';
-use POSIX 'strftime';
sub screens_ {
diff --git a/lib/VNWeb/Prelude.pm b/lib/VNWeb/Prelude.pm
index e202974c..198d09e2 100644
--- a/lib/VNWeb/Prelude.pm
+++ b/lib/VNWeb/Prelude.pm
@@ -8,7 +8,7 @@
# use Exporter 'import';
# use Time::HiRes 'time';
# use List::Util 'min', 'max', 'sum';
-# use POSIX 'ceil', 'floor';
+# use POSIX 'ceil', 'floor', 'strftime';
#
# use VNDBUtil;
# use VNDB::BBCode;
@@ -53,7 +53,7 @@ sub import {
use Exporter 'import';
use Time::HiRes 'time';
use List::Util 'min', 'max', 'sum';
- use POSIX 'ceil', 'floor';
+ use POSIX 'ceil', 'floor', 'strftime';
use VNDBUtil;
use VNDB::BBCode;
diff --git a/lib/VNWeb/ULists/List.pm b/lib/VNWeb/ULists/List.pm
index 3d2bf681..dbdfddc9 100644
--- a/lib/VNWeb/ULists/List.pm
+++ b/lib/VNWeb/ULists/List.pm
@@ -3,7 +3,6 @@ package VNWeb::ULists::Main;
use VNWeb::Prelude;
use VNWeb::ULists::Lib;
use VNWeb::Releases::Lib;
-use POSIX 'strftime';
sub opt {
diff --git a/lib/VNWeb/VN/Page.pm b/lib/VNWeb/VN/Page.pm
index ae32ecf3..ed4f6e75 100644
--- a/lib/VNWeb/VN/Page.pm
+++ b/lib/VNWeb/VN/Page.pm
@@ -4,7 +4,6 @@ use VNWeb::Prelude;
use VNWeb::Releases::Lib;
use VNWeb::Images::Lib qw/image_flagging_display image_ enrich_image_obj/;
use VNDB::Func 'fmtrating';
-use POSIX 'strftime';
# Enrich everything necessary to at least render infobox_().