summaryrefslogtreecommitdiff
path: root/lib/Multi/APIDump.pm
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2013-01-18 13:46:13 +0100
committerYorhel <git@yorhel.nl>2013-01-18 13:46:35 +0100
commit4de98aaeb5791018929f46a354c4c66fe26fed6c (patch)
tree08d1ebe0f55c9ff010834afec0d13d3151247c34 /lib/Multi/APIDump.pm
parentd7fd27b181caf06a652a36bfa6ae15e711a9e964 (diff)
Added daily-generated JSON dump of the trait information
Diffstat (limited to 'lib/Multi/APIDump.pm')
-rw-r--r--lib/Multi/APIDump.pm62
1 files changed, 50 insertions, 12 deletions
diff --git a/lib/Multi/APIDump.pm b/lib/Multi/APIDump.pm
index 22f02bad..b9723e67 100644
--- a/lib/Multi/APIDump.pm
+++ b/lib/Multi/APIDump.pm
@@ -17,11 +17,12 @@ sub spawn {
my $p = shift;
POE::Session->create(
package_states => [
- $p => [qw| _start shutdown generate tags_write |],
+ $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson|],
],
heap => {
regenerate_interval => 86400, # daily min.
- tagsfile => "$VNDB::ROOT/www/api/tags.json.gz",
+ tagsfile => "$VNDB::ROOT/www/api/tags.json.gz",
+ traitsfile => "$VNDB::ROOT/www/api/traits.json.gz",
@_,
},
);
@@ -30,19 +31,21 @@ sub spawn {
sub _start {
$_[KERNEL]->alias_set('apidump');
- $_[KERNEL]->yield('generate');
+ $_[KERNEL]->yield('tags_gen');
+ $_[KERNEL]->delay(traits_gen => 10);
$_[KERNEL]->sig(shutdown => 'shutdown');
}
sub shutdown {
- $_[KERNEL]->delay('generate');
+ $_[KERNEL]->delay('tags_gen');
+ $_[KERNEL]->delay('traits_gen');
$_[KERNEL]->alias_remove('apidump');
}
-sub generate {
- $_[KERNEL]->alarm(generate => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval});
+sub tags_gen {
+ $_[KERNEL]->alarm(tags_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval});
# The subqueries are kinda ugly, but it's convenient to have everything in a single query.
$_[KERNEL]->post(pg => query => q{
@@ -61,18 +64,53 @@ sub tags_write {
for(@$res) {
$_->{id} *= 1;
$_->{meta} = $_->{meta} ? JSON::XS::true : JSON::XS::false;
+ $_->{vns} *= 1;
$_->{aliases} = [ split /\$\$\$-\$\$\$/, ($_->{aliases}||'') ];
$_->{parents} = [ map $_*1, split /,/, ($_->{parents}||'') ];
}
- open my $f, '>:gzip:utf8', "$_[HEAP]{tagsfile}~" or die $!;
- print $f JSON::XS->new->encode($res);
+ $_[KERNEL]->yield(writejson => $res, $_[HEAP]{tagsfile}, $time, $ws);
+}
+
+
+sub traits_gen {
+ $_[KERNEL]->alarm(traits_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval});
+
+ $_[KERNEL]->post(pg => query => q{
+ SELECT id, name, alias AS aliases, description, meta, c_items AS chars,
+ (SELECT string_agg(parent::text, ',') FROM traits_parents WHERE trait = id) AS parents
+ FROM traits WHERE state = 2
+ }, undef, 'traits_write');
+}
+
+
+sub traits_write {
+ my($res, $time) = @_[ARG1,ARG3];
+ my $ws = time;
+
+ for(@$res) {
+ $_->{id} *= 1;
+ $_->{meta} = $_->{meta} ? JSON::XS::true : JSON::XS::false;
+ $_->{chars} *= 1;
+ $_->{aliases} = [ split /\r?\n/, ($_->{aliases}||'') ];
+ $_->{parents} = [ map $_*1, split /,/, ($_->{parents}||'') ];
+ }
+
+ $_[KERNEL]->yield(writejson => $res, $_[HEAP]{traitsfile}, $time, $ws);
+}
+
+
+sub writejson {
+ my($data, $file, $sqltime, $procstart) = @_[ARG0..$#_];
+
+ open my $f, '>:gzip:utf8', "$file~" or die "Writing $file: $!";
+ print $f JSON::XS->new->encode($data);
close $f;
- rename "$_[HEAP]{tagsfile}~", $_[HEAP]{tagsfile} or die $!;
+ rename "$file~", $file or die "Renaming $file: $!";
- my $wt = time-$ws;
- $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, tags: %d.',
- $_[HEAP]{tagsfile}, $time, $wt, (-s $_[HEAP]{tagsfile})/1024, scalar @$res);
+ my $wt = time-$procstart;
+ $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, items: %d.',
+ $file, $sqltime, $wt, (-s $file)/1024, scalar @$data);
}
1;