API dump: Add daily votes dump

2 files changed, 48 insertions, 1 deletions

diff --git a/data/docs/14 b/data/docs/14
index 923da28a..f53ee2f1 100644
--- a/data/docs/14
+++ b/data/docs/14
@@ -146,3 +146,17 @@
  <a href="/i112">i112</a> is often displayed as "Eyes > Green", to
  differentiate it with <a href="/i50">i50</a>, which is "Hair > Green".
 </p>
+
+
+:SUB:Votes
+<p>
+ <b>URL:</b> <a href="http://vndb.org/api/votes.gz">http://vndb.org/api/votes.gz</a><br />
+ <b>Updated:</b> Every 24 hours.<br />
+ <b>Size:</b> ~1.2 MiB compressed, ~3.5 MiB uncompressed.<br />
+ This dump contains the VN votes of all users who did not mark their vote list
+ as private. Votes from known duplicate accounts or from users who voted on
+ unreleased VNs are also not included.<br />
+ Each line in the file represents a single vote. Each line contains the VN id,
+ user ID and vote, separated by a space. Votes are as listed on the site,
+ multiplied by 10 (i.e. in the range of 10 - 100).
+</p>
diff --git a/lib/Multi/APIDump.pm b/lib/Multi/APIDump.pm
index b9723e67..3441225a 100644
--- a/lib/Multi/APIDump.pm
+++ b/lib/Multi/APIDump.pm
@@ -17,12 +17,13 @@ sub spawn {
   my $p = shift;
   POE::Session->create(
     package_states => [
-      $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson|],
+      $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson votes_gen votes_write|],
     ],
     heap => {
       regenerate_interval => 86400, # daily min.
       tagsfile   => "$VNDB::ROOT/www/api/tags.json.gz",
       traitsfile => "$VNDB::ROOT/www/api/traits.json.gz",
+      votesfile  => "$VNDB::ROOT/www/api/votes.gz",
       @_,
     },
   );
@@ -33,6 +34,7 @@ sub _start {
   $_[KERNEL]->alias_set('apidump');
   $_[KERNEL]->yield('tags_gen');
   $_[KERNEL]->delay(traits_gen => 10);
+  $_[KERNEL]->delay(votes_gen => 20);
   $_[KERNEL]->sig(shutdown => 'shutdown');
 }
 
@@ -40,6 +42,7 @@ sub _start {
 sub shutdown {
   $_[KERNEL]->delay('tags_gen');
   $_[KERNEL]->delay('traits_gen');
+  $_[KERNEL]->delay('votes_gen');
   $_[KERNEL]->alias_remove('apidump');
 }
 
@@ -113,4 +116,34 @@ sub writejson {
     $file, $sqltime, $wt, (-s $file)/1024, scalar @$data);
 }
 
+sub votes_gen {
+  $_[KERNEL]->alarm(votes_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval});
+
+  $_[KERNEL]->post(pg => query => q{
+    SELECT vv.vid||' '||vv.uid||' '||vv.vote as l
+      FROM votes vv
+      JOIN users u ON u.id = vv.uid
+      JOIN vn v ON v.id = vv.vid
+     WHERE NOT v.hidden
+       AND NOT u.ign_votes
+       AND NOT EXISTS(SELECT 1 FROM users_prefs up WHERE up.uid = u.id AND key = 'hide_list')
+  }, undef, 'votes_write');
+}
+
+
+sub votes_write {
+  my($res, $sqltime) = @_[ARG1,ARG3];
+  my $ws = time;
+
+  my $file = $_[HEAP]{votesfile};
+  open my $f, '>:gzip:utf8', "$file~" or die "Writing $file: $!";
+  printf $f "%s\n", $_->{l} for (@$res);
+  close $f;
+  rename "$file~", $file or die "Renaming $file: $!";
+
+  my $wt = time-$ws;
+  $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, items: %d.',
+    $file, $sqltime, $wt, (-s $file)/1024, scalar @$res);
+}
+
 1;