From 225f58c043446d3519adc47d72dba582cb75c56f Mon Sep 17 00:00:00 2001 From: yorhel Date: Thu, 5 Jun 2008 16:20:29 +0000 Subject: Improvements in the relation graph generator, and a little cleanup in the Multi code git-svn-id: svn://vndb.org/vndb@20 1fe2e327-d9db-4752-bcf7-ef0cb4a1748b --- lib/ChangeLog | 1 + lib/Multi/Anime.pm | 2 +- lib/Multi/RG.pm | 121 ++++++++++++++++++++++++++++++++++++++++------------- util/multi.pl | 21 +--------- 4 files changed, 96 insertions(+), 49 deletions(-) diff --git a/lib/ChangeLog b/lib/ChangeLog index 0ac9e7fb..8570477b 100644 --- a/lib/ChangeLog +++ b/lib/ChangeLog @@ -13,6 +13,7 @@ TODO: 1.17 - 2008-05-23 - Added PS3 and Xbox 360 to platforms + - Relation graph generation improvements: Unicode, anti-aliassing, async 1.16 - 2008-05-22 - Release dates in the current year or month without a specified day will diff --git a/lib/Multi/Anime.pm b/lib/Multi/Anime.pm index 0da54e2e..68fcce4e 100644 --- a/lib/Multi/Anime.pm +++ b/lib/Multi/Anime.pm @@ -7,7 +7,7 @@ package Multi::Anime; use strict; use warnings; -use POE 'Wheel::UDP'; +use POE 'Wheel::UDP', 'Filter::Stream'; use Tie::ShareLite ':lock'; use Socket 'inet_ntoa'; use Time::HiRes 'time'; diff --git a/lib/Multi/RG.pm b/lib/Multi/RG.pm index ebb93013..51bddc32 100644 --- a/lib/Multi/RG.pm +++ b/lib/Multi/RG.pm @@ -7,23 +7,34 @@ package Multi::RG; use strict; use warnings; -use POE; +use POE 'Wheel::Run', 'Filter::Stream'; +use Encode 'encode_utf8'; use Text::Unidecode; use GraphViz; +# GraphViz.pm is only used to create the layout, the actual +# 'dot' command is manually run using POE::Wheel::Run +# TODO: Get rid of GraphViz.pm altogether? (it does suck...) + + sub spawn { my $p = shift; POE::Session->create( package_states => [ - $p => [qw| _start cmd_relgraph creategraph getrel relscomplete buildgraph graphcomplete |], + $p => [qw| + _start cmd_relgraph + creategraph getrel relscomplete buildgraph savegraph completegraph + proc_stdin proc_stdout proc_stderr proc_closed proc_child + |], ], heap => { - font => 's', + font => 'Arial', fsize => [ 9, 7, 10 ], # nodes, edges, node_title imgdir => '/www/vndb/static/rg', datdir => '/www/vndb/data/rg', moy => [qw| Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec |], + dot => '/usr/bin/dot', @_, } ); @@ -32,6 +43,7 @@ sub spawn { sub _start { $_[KERNEL]->alias_set('rg'); + $_[KERNEL]->sig(CHLD => 'proc_child'); $_[KERNEL]->call(core => register => qr/^relgraph ((?:[0-9]+)(?:\s+[0-9]+)*|all)$/, 'cmd_relgraph'); # regenerate all relation graphs once a month @@ -62,10 +74,11 @@ sub creategraph { # id # getrel (recursive) # relscomplete # if !rels - # graphcomplete + # completegraph # else # buildgraph - # graphcomplete + # savegraph + # completegraph $_[KERNEL]->call(core => log => 3, 'Processing graph for v%d', $_[ARG0]); $_[HEAP]{gv} = GraphViz->new( @@ -97,11 +110,8 @@ sub getrel { # vid ); $s->execute($_[ARG0], $_[ARG0]); while(my $r = $s->fetchrow_hashref) { - if($r->{vid1} < $r->{vid2}) { - $_[HEAP]{rels}{$r->{vid1}.'-'.$r->{vid2}} = reverserel($r->{relation}); - } else { - $_[HEAP]{rels}{$r->{vid2}.'-'.$r->{vid1}} = $r->{relation} if $r->{vid1} < $r->{vid2}; - } + $_[HEAP]{rels}{$r->{vid1}.'-'.$r->{vid2}} = reverserel($r->{relation}) if $r->{vid1} < $r->{vid2}; + $_[HEAP]{rels}{$r->{vid2}.'-'.$r->{vid1}} = $r->{relation} if $r->{vid1} > $r->{vid2}; for (1,2) { my($vid, $title, $date, $lang) = @$r{ "vid$_", "title$_", "date$_", "lang$_" }; @@ -122,7 +132,7 @@ sub relscomplete { # heap->nodes and heap->rels are now assumed to contain all n $_[KERNEL]->call(core => log => 3, 'No relation graph for v%d', $_[HEAP]{vid}); $Multi::SQL->do('UPDATE vn SET rgraph = 0 WHERE id = ?', undef, $_[HEAP]{vid}); $_[HEAP]{nodes}{$_[HEAP]{vid}} = []; - $_[KERNEL]->yield('graphcomplete'); + $_[KERNEL]->yield('completegraph'); return; } $_[KERNEL]->call(core => log => 3, 'Fetched all relation data'); @@ -132,7 +142,7 @@ sub relscomplete { # heap->nodes and heap->rels are now assumed to contain all n my $date = sprintf '%08d', $_->[2]; $date =~ s#^([0-9]{4})([0-9]{2}).+#$1==0?'N/A':$1==9999?'TBA':(($2&&$2<13?($_[HEAP]{moy}[$2-1].' '):'').$1)#e; - my $title = unidecode($_->[1]); + my $title = $_->[1]; $title = substr($title, 0, 27).'...' if length($title) > 30; $title =~ s/&/&/g; @@ -142,13 +152,13 @@ sub relscomplete { # heap->nodes and heap->rels are now assumed to contain all n fontsize => $_[HEAP]{fsize}[0], style => 'setlinewidth(0.5)', URL => '/v'.$_->[0], - tooltip => $title, + tooltip => encode_utf8($_->[1]), label => sprintf( '< - +
%s
%s
%s %s
>', - $_[HEAP]{fsize}[2], $title, $date, $_->[3]||'N/A' + $_[HEAP]{fsize}[2], encode_utf8($title), $date, $_->[3]||'N/A' ), ); } @@ -198,33 +208,53 @@ sub buildgraph { my $gid = $Multi::SQL->prepare("SELECT nextval('relgraph_seq')"); $gid->execute; $gid = $gid->fetchrow_arrayref->[0]; - my $gif = sprintf '%s/%02d/%d.gif', $_[HEAP]{imgdir}, $gid % 100, $gid; - my $cmap = sprintf '%s/%02d/%d.cmap', $_[HEAP]{datdir}, $gid % 100, $gid; + $_[HEAP]{gid} = [ + $gid, + sprintf('%s/%02d/%d.gif', $_[HEAP]{imgdir}, $gid % 100, $gid), + sprintf('%s/%02d/%d.cmap', $_[HEAP]{datdir}, $gid % 100, $gid) + ]; + + # roughly equivalent to: + # cat layout.txt | dot -Tgif -o graph.gif -Tcmap > graph.cmap + $_[HEAP]{proc} = POE::Wheel::Run->new( + Program => $_[HEAP]{dot}, + ProgramArgs => [ '-Tgif', '-o', $_[HEAP]{gid}[1], '-Tcmap' ], + StdioFilter => POE::Filter::Stream->new(), + StdinEvent => 'proc_stdin', + StdoutEvent => 'proc_stdout', + StderrEvent => 'proc_stderr', + CloseEvent => 'proc_closed', + ); + $_[HEAP]{proc}->put($_[HEAP]{gv}->as_debug); + $_[HEAP]{cmap} = ''; +} - # generate the graph - $_[HEAP]{gv}->as_gif($gif); - chmod 0666, $gif; - # generate the image map - open my $F, '>', $cmap or die $!; +sub savegraph { + # save the image map + open my $F, '>', $_[HEAP]{gid}[2] or die $!; print $F '\n"; - (my $d = $_[HEAP]{gv}->as_cmapx) =~ s/(id|name)="[^"]+"/$1="rgraph"/g; - print $F $d; + print $F ''."\n"; + print $F $_[HEAP]{cmap}; + print $F ''; close $F; - chmod 0666, $cmap; + + # proper chmod + chmod 0666, $_[HEAP]{gid}[2]; + chmod 0666, $_[HEAP]{gid}[1]; # update the VN table $Multi::SQL->do(sprintf q| UPDATE vn SET rgraph = %d WHERE id IN(%s)|, - $gid, join(',', keys %{$_[HEAP]{nodes}})); + $_[HEAP]{gid}[0], join(',', keys %{$_[HEAP]{nodes}})); - $_[KERNEL]->yield('graphcomplete'); + $_[KERNEL]->yield('completegraph'); } -sub graphcomplete { # all actions to create the graph (after calling creategraph) are now done +sub completegraph { $_[KERNEL]->call(core => log => 3, 'Generated the relation graph for v%d', $_[HEAP]{vid}); # remove processed vns, and check for other graphs in the queue @@ -233,13 +263,33 @@ sub graphcomplete { # all actions to create the graph (after calling creategraph $_[KERNEL]->yield(creategraph => $_[HEAP]{todo}[0]); } else { $_[KERNEL]->post(core => finish => $_[HEAP]{curcmd}); - delete @{$_[HEAP]}{qw| vid nodes rels curcmd gv todo |}; + delete @{$_[HEAP]}{qw| vid nodes rels curcmd gv todo gid cmap |}; } } +# POE handlers for communication with GraphViz +sub proc_stdin { + $_[HEAP]{proc}->shutdown_stdin; +} +sub proc_stdout { + $_[HEAP]{cmap} .= $_[ARG0]; +} +sub proc_stderr { + $_[KERNEL]->call(core => log => 1, 'GraphViz STDERR: %s', $_[ARG0]); +} +sub proc_closed { + $_[KERNEL]->yield('savegraph'); + undef $_[HEAP]{proc}; +} +sub proc_child { + 1; # do nothing, just make sure SIGCHLD is handled to reap the process +} + + + # Not a POE handler, just a small macro sub reverserel { # relation @@ -249,3 +299,16 @@ sub reverserel { # relation 1; +__END__ + + + # generate the graph + #$_[HEAP]{gv}->as_gif($gif); + #chmod 0666, $gif; + + #print $_[HEAP]{gv}->as_debug; + + # postscript output (experimental) + #$_[HEAP]{gv}->as_svg('/tmp/test.svg'); + # system('/usr/bin/inkscape --export-png '.$gif.' /tmp/test.svg'); + # chmod 0666, $gif; diff --git a/util/multi.pl b/util/multi.pl index 888dbd4f..d45150fb 100644 --- a/util/multi.pl +++ b/util/multi.pl @@ -1,24 +1,8 @@ #!/usr/bin/perl -# This is just a small script to test and play around with a -# processing queue for actions on VNDB that do not have a -# strict time limit. i.e. resizing and optimizing cover images -# and (re)generating the relation graphs. Because I'm using -# the POE framework, it will also be possible to integrate -# Multi the IRC bot in the same process. -# -# The queue is an array of commands, and should be executed -# in chronological order. Commands are in the form of -# [cmd] [arguments] -# where [cmd] is an internal command, and [arguments] a -# whitespace seperated list of arguments. -# -# Commands can be added from the web interface using shared -# memory, or from IRC if Multi is going to integrated in here. - # Usage: -# ./multi.pl [-c] [-s] [cmd1] [cmd2] .. +# ./multi.pl [-c] [-s] [-a] [cmd1] [cmd2] .. # -c Do not daemonize, just execute the commands specified # on the command line and exit. # -s Same as -c, but also execute commands in the shared @@ -51,7 +35,6 @@ use Multi::IRC; BEGIN { require 'global.pl' } - $ENV{PATH} = '/usr/bin'; our $LOGDIR = '/www/vndb/data/log'; our $LOGLVL = 3; # 3:DEBUG, 2:ACTIONS, 1:WARN our $STOP = 0; @@ -77,7 +60,7 @@ Multi::Core->spawn(); Multi::RG->spawn(); Multi::Image->spawn(); Multi::Sitemap->spawn(); -Multi::Anime->spawn(); +Multi::Anime->spawn() if !$VNDB::DEBUG; # no need to update anime from the beta Multi::Maintenance->spawn(); Multi::IRC->spawn() if !$VNDB::DEBUG; -- cgit v1.2.3