diff options
author | yorhel <yorhel@1fe2e327-d9db-4752-bcf7-ef0cb4a1748b> | 2008-04-26 09:25:45 +0000 |
---|---|---|
committer | yorhel <yorhel@1fe2e327-d9db-4752-bcf7-ef0cb4a1748b> | 2008-04-26 09:25:45 +0000 |
commit | 88853c1168a333c7c7b7951b2931852e6df7da55 (patch) | |
tree | fbad6f17f0820c89ce5dd8da748cf37b56549d14 /lib/Multi | |
parent | d2efb163c11a11e02abd7251fdfde3cb54c710ef (diff) |
Stupid comments - like I can keep track of everything I change...
git-svn-id: svn://vndb.org/vndb@5 1fe2e327-d9db-4752-bcf7-ef0cb4a1748b
Diffstat (limited to 'lib/Multi')
-rw-r--r-- | lib/Multi/Core.pm | 29 | ||||
-rw-r--r-- | lib/Multi/IRC.pm | 16 | ||||
-rw-r--r-- | lib/Multi/Image.pm | 119 | ||||
-rw-r--r-- | lib/Multi/Maintenance.pm | 84 | ||||
-rw-r--r-- | lib/Multi/RG.pm | 3 | ||||
-rw-r--r-- | lib/Multi/Sitemap.pm | 4 |
6 files changed, 239 insertions, 16 deletions
diff --git a/lib/Multi/Core.pm b/lib/Multi/Core.pm index 1eea66ca..6f3b452c 100644 --- a/lib/Multi/Core.pm +++ b/lib/Multi/Core.pm @@ -7,19 +7,20 @@ package Multi::Core; use strict; use warnings; -use POE; +use POE 'Component::Cron'; use Storable 'freeze', 'thaw'; use IPC::ShareLite ':lock'; use Time::HiRes 'time', 'gettimeofday', 'tv_interval'; # overload time() +use DateTime::Event::Cron; # bug in PoCo::Cron sub spawn { my $p = shift; POE::Session->create( package_states => [ - $p => [qw| _start register fetch queue execute finish log cmd_exit |], + $p => [qw| _start register addcron fetch queue execute finish log cmd_exit |], ], - heap => { queue => [], cmds => [], running => 0, starttime => 0 }, + heap => { cron => [], queue => [], cmds => [], running => 0, starttime => 0 }, ); } @@ -39,6 +40,13 @@ sub register { # regex, state } +sub addcron { # cronline, cmd + return if $Multi::DAEMONIZE; # no cronjobs when we aren't a daemon! + push @{$_[HEAP]{cron}}, POE::Component::Cron->from_cron($_[ARG0], $_[SESSION], queue => $_[ARG1]); + $_[KERNEL]->call(core => log => 3, "Added cron: %s %s", $_[ARG0], $_[ARG1]); +} + + sub fetch { # lastfetch my $s = IPC::ShareLite->new(-key => $VNDB::SHMKEY,-create => 1, -destroy => 0); $s->lock(LOCK_SH); @@ -97,12 +105,18 @@ sub finish { # cmd [, stop ] sub log { # level, msg return if $_[ARG0] > $Multi::LOGLVL; - #open(my $F, \&STDOUT); #'>>', $Multi::LOGDIR.'/multi.log'); + (my $p = $_[SENDER][2]{$_[CALLER_STATE]}[0]) =~ s/^Multi:://; # NOT PORTABLE - printf "[%s] (%s) %s::%s: %s\n", scalar localtime, + my $msg = sprintf '(%s) %s::%s: %s', (qw|WRN ACT DBG|)[$_[ARG0]-1], $p, $_[CALLER_STATE], $_[ARG2] ? sprintf($_[ARG1], @_[ARG2..$#_]) : $_[ARG1]; - #close $F; + + open(my $F, '>>', $Multi::LOGDIR.'/multi.log'); + printf $F "[%s] %s\n", scalar localtime, $msg; + close $F; + + # (debug) log to stdout as well... + #printf "[%s] %s\n", scalar localtime, $msg; } @@ -118,7 +132,8 @@ sub cmd_exit { $s->unlock(); undef $s; - $_[KERNEL]->delay('fetch'); # This'll make the current session stop + $_[KERNEL]->delay('fetch'); # stop fetching + $_->delete() for (@{$_[HEAP]{cron}}); # stop scheduling cron jobs $_[KERNEL]->signal($_[KERNEL], 'shutdown'); # Broadcast to other sessions } diff --git a/lib/Multi/IRC.pm b/lib/Multi/IRC.pm index 3ab4790a..75300419 100644 --- a/lib/Multi/IRC.pm +++ b/lib/Multi/IRC.pm @@ -109,15 +109,15 @@ sub irc_msg { my $m = $_[ARG2]; if($m =~ /^say (.+)$/) { - $_[KERNEL]->post(circ => privmsg => $_[HEAP]{o}{channel}, $1); } - elsif($m =~ /^me (.+)$/) { - $_[KERNEL]->post(circ => ctcp => $_[HEAP]{o}{channel}, "ACTION $1"); } - elsif($m =~ /^cmd (.+)$/) { - $_[KERNEL]->post(core => queue => $1); } - elsif($m =~ /^eval (.+)$/) { + $_[KERNEL]->post(circ => privmsg => $_[HEAP]{o}{channel}, $1); + } elsif($m =~ /^me (.+)$/) { + $_[KERNEL]->post(circ => ctcp => $_[HEAP]{o}{channel}, "ACTION $1"); + } elsif($m =~ /^cmd (.+)$/) { + $_[KERNEL]->post(core => queue => $1); + } elsif($m =~ /^eval (.+)$/) { $_[KERNEL]->post(circ => privmsg => $nick, 'eval: '.$_) - for (split /\r?\n/, eval($1)||$@); } - else { + for (split /\r?\n/, eval($1)||$@); + } else { $_[KERNEL]->post(circ => privmsg => $nick, 'Unkown command'); } # TODO: add command to view the current queue, and a method to send log messages diff --git a/lib/Multi/Image.pm b/lib/Multi/Image.pm new file mode 100644 index 00000000..f9ccd83e --- /dev/null +++ b/lib/Multi/Image.pm @@ -0,0 +1,119 @@ + +# +# Multi::Image - Image compressing and resizing +# + +package Multi::Image; + +use strict; +use warnings; +use POE; +use Image::Magick; +use Image::MetaData::JPEG; + + +sub spawn { + my $p = shift; + POE::Session->create( + package_states => [ + $p => [qw| _start cmd_coverimage format compress update finish |], + ], + heap => { + imgpath => '/www/vndb/static/cv' + }, + ); +} + + +sub _start { + $_[KERNEL]->alias_set('image'); + $_[KERNEL]->sig(shutdown => 'shutdown'); + $_[KERNEL]->call(core => register => qr/^coverimage(?: ([0-9]+)|)$/, 'cmd_coverimage'); + + # check for unprocessed cover images every day on 0:00 and 12:00 local time + $_[KERNEL]->post(core => addcron => '0 0,12 * * *', 'coverimage'); +} + + +sub cmd_coverimage { + $_[HEAP]{curcmd} = $_[ARG0]; + + if($_[ARG1]) { + $_[HEAP]{todo} = [ $_[ARG1] ]; + } else { + my $q = $Multi::SQL->prepare('SELECT image FROM vn_rev WHERE image < 0'); + $q->execute(); + $_[HEAP]{todo} = [ map { -1*$_->[0]} @{$q->fetchall_arrayref([])} ]; + if(!@{$_[HEAP]{todo}}) { + $_[KERNEL]->call(core => log => 2, 'No images to process'); + $_[KERNEL]->yield('finish'); + return; + } + } + $_[KERNEL]->yield(format => $_[HEAP]{todo}[0]); +} + + +sub format { # imgid + $_[HEAP]{imgid} = $_[ARG0]; + $_[HEAP]{img} = sprintf '%s/%02d/%d.jpg', $_[HEAP]{imgpath}, $_[ARG0]%50, $_[ARG0]; + $_[KERNEL]->call(core => log => 3, 'Processing image %d', $_[HEAP]{imgid}); + + $_[HEAP]{im} = Image::Magick->new; + $_[HEAP]{im}->Read($_[HEAP]{img}); + $_[HEAP]{im}->Set(magick => 'JPEG'); + my($w, $h) = ($_[HEAP]{im}->Get('width'), $_[HEAP]{im}->Get('height')); + if($w > 256 || $h > 400) { + $_[KERNEL]->call(core => log => 3, 'Image too large (%dx%d), resizing', $w, $h); + if($w/$h > 256/400) { # width is the limiting factor + $h *= 256/$w; + $w = 256; + } else { + $w *= 400/$h; + $h = 400; + } + $_[HEAP]{im}->Thumbnail(width => $w, height => $h); + } + + $_[KERNEL]->yield('compress'); +} + + +sub compress { + $_[HEAP]{im}->Set(quality => 80); + $_[HEAP]{im}->Write($_[HEAP]{img}); + undef $_[HEAP]{im}; + + $_[HEAP]{md} = Image::MetaData::JPEG->new($_[HEAP]{img}); + $_[HEAP]{md}->drop_segments('METADATA'); + $_[HEAP]{md}->save($_[HEAP]{img}); + undef $_[HEAP]{md}; + + $_[KERNEL]->call(core => log => 3, 'Compressed image %d to %.2fkB', $_[HEAP]{imgid}, (-s $_[HEAP]{img})/1024); + $_[KERNEL]->yield('update'); +} + + +sub update { + $Multi::SQL->do('UPDATE vn_rev SET image = ? WHERE image = ?', undef, $_[HEAP]{imgid}, -1*$_[HEAP]{imgid}); + + $_[KERNEL]->yield('finish'); +} + + +sub finish { + if($_[HEAP]{imgid}) { + $_[HEAP]{todo} = [ grep { $_[HEAP]{imgid} != $_ } @{$_[HEAP]{todo}} ]; + if(@{$_[HEAP]{todo}}) { + $_[KERNEL]->yield(format => $_[HEAP]{todo}[0]); + return; + } + } + + $_[KERNEL]->post(core => finish => $_[HEAP]{curcmd}); + delete @{$_[HEAP]}{qw| curcmd imgid img im md todo |}; +} + + +1; + diff --git a/lib/Multi/Maintenance.pm b/lib/Multi/Maintenance.pm new file mode 100644 index 00000000..fb74e192 --- /dev/null +++ b/lib/Multi/Maintenance.pm @@ -0,0 +1,84 @@ + +# +# Multi::Maintenance - General maintenance functions +# + +package Multi::Maintenance; + +use strict; +use warnings; +use POE; + + +sub spawn { + my $p = shift; + POE::Session->create( + package_states => [ + $p => [qw| _start cmd_maintenance vncache ratings prevcache integrity |], + ], + ); +} + + +sub _start { + $_[KERNEL]->alias_set('maintenance'); + $_[KERNEL]->call(core => register => qr/^maintenance((?: (?:all|vncache|ratings|prevcache|integrity))+)$/, 'cmd_maintenance'); + + # Perform all maintenance functions every day on 0:00 + $_[KERNEL]->post(core => addcron => '0 0 * * *', 'maintenance all'); +} + + +sub cmd_maintenance { + local $_ = $_[ARG1]; + + $_[KERNEL]->yield('vncache') if /(vncache|all)/; + $_[KERNEL]->yield('ratings') if /(ratings|all)/; + $_[KERNEL]->yield('prevcache') if /(prevcache|all)/; + $_[KERNEL]->yield('integrity') if /(integrity|all)/; + + $_[KERNEL]->post(core => finish => $_[ARG0]); +} + + +sub vncache { + $_[KERNEL]->call(core => log => 3 => 'Updating c_* columns in the vn table...'); + $Multi::SQL->do('SELECT update_vncache(0)'); +} + + +sub ratings { + $_[KERNEL]->call(core => log => 3 => 'Recalculating VN ratings...'); + $Multi::SQL->do('SELECT calculate_rating()'); +} + + +sub prevcache { + $_[KERNEL]->call(core => log => 3 => 'Updating prev column in the changes table...'); + $Multi::SQL->do(q|SELECT update_prev('vn', ''), update_prev('releases', ''), update_prev('producers', '')|); +} + + +sub integrity { + my $q = $Multi::SQL->prepare(q| + SELECT 'r', id FROM releases_rev rr + WHERE NOT EXISTS(SELECT 1 FROM releases_vn rv WHERE rr.id = rv.rid) + UNION + SELECT c.type::varchar, id FROM changes c + WHERE (c.type = 0 AND NOT EXISTS(SELECT 1 FROM vn_rev vr WHERE vr.id = c.id)) + OR (c.type = 1 AND NOT EXISTS(SELECT 1 FROM releases_rev rr WHERE rr.id = c.id)) + OR (c.type = 2 AND NOT EXISTS(SELECT 1 FROM producers_rev pr WHERE pr.id = c.id))|); + $q->execute(); + my $r = $q->fetchall_arrayref([]); + if(@$r) { + $_[KERNEL]->call(core => log => 1, '!DATABASE INCONSISTENCIES FOUND!: %s', + join(', ', map { $_->[0].':'.$_->[1] } @$r)); + } else { + $_[KERNEL]->call(core => log => 3, 'No database inconsistencies found'); + } +} + + +1; + + diff --git a/lib/Multi/RG.pm b/lib/Multi/RG.pm index 9133a7a6..bf0ad9f6 100644 --- a/lib/Multi/RG.pm +++ b/lib/Multi/RG.pm @@ -33,6 +33,9 @@ sub spawn { sub _start { $_[KERNEL]->alias_set('rg'); $_[KERNEL]->call(core => register => qr/^relgraph ((?:[0-9]+)(?:\s+[0-9]+)*|all)$/, 'cmd_relgraph'); + + # regenerate all relation graphs twice a month (every 1st and 15nd day at 03:00) + $_[KERNEL]->post(core => addcron => '0 3 1,15 * *', 'relgraph all'); } diff --git a/lib/Multi/Sitemap.pm b/lib/Multi/Sitemap.pm index 836fce5b..24981910 100644 --- a/lib/Multi/Sitemap.pm +++ b/lib/Multi/Sitemap.pm @@ -31,7 +31,9 @@ sub spawn { sub _start { $_[KERNEL]->alias_set('sitemap'); $_[KERNEL]->call(core => register => qr/^sitemap$/, 'cmd_sitemap'); - # TODO: add an event to run cmd_sitemap on a daily basis + + # Regenerate the sitemap every day on 0:00 + $_[KERNEL]->post(core => addcron => '0 0 * * *', 'sitemap'); } |