package Multi::JList; use strict; use warnings; use Multi::Core; use AnyEvent::HTTP; my %C = ( jbox => 'https://www.jbox.com/', jlist => 'https://www.jlist.com/', clean_timeout => 48*3600, check_timeout => 10*60, # Minimum time between fetches. ); sub run { shift; $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; %C = (%C, @_); push_watcher schedule 0, $C{clean_timeout}, sub { pg_cmd 'DELETE FROM shop_jlist WHERE id NOT IN(SELECT l_jlist FROM releases WHERE NOT hidden)'; }; push_watcher schedule 0, $C{check_timeout}, sub { pg_cmd q{ INSERT INTO shop_jlist (id) SELECT DISTINCT l_jlist FROM releases WHERE NOT hidden AND l_jlist <> '' AND NOT EXISTS(SELECT 1 FROM shop_jlist WHERE id = l_jlist) }, [], \&sync } } sub trysite { my($jbox, $id) = @_; my $ts = AE::now; my $url = ($jbox eq 't' ? $C{jbox} : $C{jlist}).$id; http_get $url, headers => {'User-Agent' => $C{ua} }, timeout => 60, sub { data($jbox, AE::now-$ts, $id, @_) }; } sub data { my($jbox, $time, $id, $body, $hdr) = @_; my $prefix = sprintf '[%.1fs] %s', $time, $id; return AE::log warn => "$prefix ERROR: $hdr->{Status} $hdr->{Reason}" if $hdr->{Status} !~ /^2/ && $hdr->{Status} ne '404'; return AE::log warn => "$prefix ERROR: Blocked by StackPath" if $body =~ /StackPath/; my $found = $hdr->{Status} ne '404' && $body =~ /fancybox mainProductImage/; my $outofstock = $body =~ /