diff options
Diffstat (limited to 'lib/Multi/APIDump.pm')
-rw-r--r-- | lib/Multi/APIDump.pm | 62 |
1 files changed, 50 insertions, 12 deletions
diff --git a/lib/Multi/APIDump.pm b/lib/Multi/APIDump.pm index 22f02bad..b9723e67 100644 --- a/lib/Multi/APIDump.pm +++ b/lib/Multi/APIDump.pm @@ -17,11 +17,12 @@ sub spawn { my $p = shift; POE::Session->create( package_states => [ - $p => [qw| _start shutdown generate tags_write |], + $p => [qw| _start shutdown tags_gen tags_write traits_gen traits_write writejson|], ], heap => { regenerate_interval => 86400, # daily min. - tagsfile => "$VNDB::ROOT/www/api/tags.json.gz", + tagsfile => "$VNDB::ROOT/www/api/tags.json.gz", + traitsfile => "$VNDB::ROOT/www/api/traits.json.gz", @_, }, ); @@ -30,19 +31,21 @@ sub spawn { sub _start { $_[KERNEL]->alias_set('apidump'); - $_[KERNEL]->yield('generate'); + $_[KERNEL]->yield('tags_gen'); + $_[KERNEL]->delay(traits_gen => 10); $_[KERNEL]->sig(shutdown => 'shutdown'); } sub shutdown { - $_[KERNEL]->delay('generate'); + $_[KERNEL]->delay('tags_gen'); + $_[KERNEL]->delay('traits_gen'); $_[KERNEL]->alias_remove('apidump'); } -sub generate { - $_[KERNEL]->alarm(generate => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval}); +sub tags_gen { + $_[KERNEL]->alarm(tags_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval}); # The subqueries are kinda ugly, but it's convenient to have everything in a single query. $_[KERNEL]->post(pg => query => q{ @@ -61,18 +64,53 @@ sub tags_write { for(@$res) { $_->{id} *= 1; $_->{meta} = $_->{meta} ? JSON::XS::true : JSON::XS::false; + $_->{vns} *= 1; $_->{aliases} = [ split /\$\$\$-\$\$\$/, ($_->{aliases}||'') ]; $_->{parents} = [ map $_*1, split /,/, ($_->{parents}||'') ]; } - open my $f, '>:gzip:utf8', "$_[HEAP]{tagsfile}~" or die $!; - print $f JSON::XS->new->encode($res); + $_[KERNEL]->yield(writejson => $res, $_[HEAP]{tagsfile}, $time, $ws); +} + + +sub traits_gen { + $_[KERNEL]->alarm(traits_gen => int((time+3)/$_[HEAP]{regenerate_interval}+1)*$_[HEAP]{regenerate_interval}); + + $_[KERNEL]->post(pg => query => q{ + SELECT id, name, alias AS aliases, description, meta, c_items AS chars, + (SELECT string_agg(parent::text, ',') FROM traits_parents WHERE trait = id) AS parents + FROM traits WHERE state = 2 + }, undef, 'traits_write'); +} + + +sub traits_write { + my($res, $time) = @_[ARG1,ARG3]; + my $ws = time; + + for(@$res) { + $_->{id} *= 1; + $_->{meta} = $_->{meta} ? JSON::XS::true : JSON::XS::false; + $_->{chars} *= 1; + $_->{aliases} = [ split /\r?\n/, ($_->{aliases}||'') ]; + $_->{parents} = [ map $_*1, split /,/, ($_->{parents}||'') ]; + } + + $_[KERNEL]->yield(writejson => $res, $_[HEAP]{traitsfile}, $time, $ws); +} + + +sub writejson { + my($data, $file, $sqltime, $procstart) = @_[ARG0..$#_]; + + open my $f, '>:gzip:utf8', "$file~" or die "Writing $file: $!"; + print $f JSON::XS->new->encode($data); close $f; - rename "$_[HEAP]{tagsfile}~", $_[HEAP]{tagsfile} or die $!; + rename "$file~", $file or die "Renaming $file: $!"; - my $wt = time-$ws; - $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, tags: %d.', - $_[HEAP]{tagsfile}, $time, $wt, (-s $_[HEAP]{tagsfile})/1024, scalar @$res); + my $wt = time-$procstart; + $_[KERNEL]->call(core => log => 'Wrote %s in %.2fs query + %.2fs write, size: %.1fkB, items: %d.', + $file, $sqltime, $wt, (-s $file)/1024, scalar @$data); } 1; |