diff options
author | Yorhel <git@yorhel.nl> | 2014-08-21 09:29:55 +0200 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2014-08-21 09:29:55 +0200 |
commit | a49faf37b12a326f232420089b2557ca957b2f43 (patch) | |
tree | 9f3bb3481959c8d5fa6796ae8acb67690a2d5f62 | |
parent | ffd93bc792f7bb43f3a4bb054b402d90f309e094 (diff) |
API dump: Add daily votes dump
-rw-r--r-- | data/docs/14 | 14 | ||||
-rw-r--r-- | lib/Multi/APIDump.pm | 35 |
2 files changed, 48 insertions, 1 deletions
diff --git a/data/docs/14 b/data/docs/14 index 923da28a..f53ee2f1 100644 --- a/data/docs/14 +++ b/data/docs/14 @@ -146,3 +146,17 @@ <a href="/i112">i112</a> is often displayed as "Eyes > Green", to differentiate it with <a href="/i50">i50</a>, which is "Hair > Green". </p> + + +:SUB:Votes +<p> + <b>URL:</b> <a href="http://vndb.org/api/votes.gz">http://vndb.org/api/votes.gz</a><br /> + <b>Updated:</b> Every 24 hours.<br /> + <b>Size:</b> ~1.2 MiB compressed, ~3.5 MiB uncompressed.<br /> + This dump contains the VN votes of all users who did not mark their vote list + as private. Votes from known duplicate accounts or from users who voted on + unreleased VNs are also not included.<br /> + Each line in the file represents a single vote. Each line contains the VN id, + user ID and vote, separated by a space. Votes are as listed on the site, + multiplied by 10 (i.e. in the range of 10 - 100). +</p> diff --git a/lib/Multi/APIDump.pm b/lib/Multi/APIDump.pm index b9723e67..3441225a 100644 --- a/lib/Multi/APIDump.pm +++ b/lib/Multi/APIDump.pm @@ -17,12 +17,13 @@ sub spawn { my $p = shift; POE::Session->create( package_states => [ - $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson|], + $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson votes_gen votes_write|], ], heap => { regenerate_interval => 86400, # daily min. tagsfile => "$VNDB::ROOT/www/api/tags.json.gz", traitsfile => "$VNDB::ROOT/www/api/traits.json.gz", + votesfile => "$VNDB::ROOT/www/api/votes.gz", @_, }, ); @@ -33,6 +34,7 @@ sub _start { $_[KERNEL]->alias_set('apidump'); $_[KERNEL]->yield('tags_gen'); $_[KERNEL]->delay(traits_gen => 10); + $_[KERNEL]->delay(votes_gen => 20); $_[KERNEL]->sig(shutdown => 'shutdown'); } @@ -40,6 +42,7 @@ sub _start { sub shutdown { $_[KERNEL]->delay('tags_gen'); $_[KERNEL]->delay('traits_gen'); + $_[KERNEL]->delay('votes_gen'); $_[KERNEL]->alias_remove('apidump'); } @@ -113,4 +116,34 @@ sub writejson { $file, $sqltime, $wt, (-s $file)/1024, scalar @$data); } +sub votes_gen { + $_[KERNEL]->alarm(votes_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval}); + + $_[KERNEL]->post(pg => query => q{ + SELECT vv.vid||' '||vv.uid||' '||vv.vote as l + FROM votes vv + JOIN users u ON u.id = vv.uid + JOIN vn v ON v.id = vv.vid + WHERE NOT v.hidden + AND NOT u.ign_votes + AND NOT EXISTS(SELECT 1 FROM users_prefs up WHERE up.uid = u.id AND key = 'hide_list') + }, undef, 'votes_write'); +} + + +sub votes_write { + my($res, $sqltime) = @_[ARG1,ARG3]; + my $ws = time; + + my $file = $_[HEAP]{votesfile}; + open my $f, '>:gzip:utf8', "$file~" or die "Writing $file: $!"; + printf $f "%s\n", $_->{l} for (@$res); + close $f; + rename "$file~", $file or die "Renaming $file: $!"; + + my $wt = time-$ws; + $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, items: %d.', + $file, $sqltime, $wt, (-s $file)/1024, scalar @$res); +} + 1; |