diff options
author | Yorhel <git@yorhel.nl> | 2020-11-03 13:30:17 +0100 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2020-11-03 13:30:55 +0100 |
commit | 3e81dbd2dd55d32518e756491dedfeccfaa6ce41 (patch) | |
tree | bc0bee41dd0ab30ece692e48e828f384bd77f046 /lib | |
parent | f3420b701aa033c90678216bd36d374c10e3ac9f (diff) |
Multi::Denpa: Update affiliate crawler to use their new site
Diffstat (limited to 'lib')
-rw-r--r-- | lib/Multi/Denpa.pm | 34 | ||||
-rw-r--r-- | lib/VNDB/Config.pm | 2 | ||||
-rw-r--r-- | lib/VNDB/ExtLinks.pm | 2 |
3 files changed, 14 insertions, 24 deletions
diff --git a/lib/Multi/Denpa.pm b/lib/Multi/Denpa.pm index bdecd085..6076adc1 100644 --- a/lib/Multi/Denpa.pm +++ b/lib/Multi/Denpa.pm @@ -4,16 +4,11 @@ use strict; use warnings; use Multi::Core; use AnyEvent::HTTP; -use JSON::XS 'decode_json'; -use MIME::Base64 'encode_base64'; use VNDB::Config; -use TUWF::Misc 'uri_escape'; +use VNDB::ExtLinks (); my %C = ( - api => '', - user => '', - pass => '', clean_timeout => 48*3600, check_timeout => 15*60, ); @@ -44,24 +39,20 @@ sub data { my $prefix = sprintf '[%.1fs] %s', $time, $id; return AE::log warn => "$prefix ERROR: $hdr->{Status} $hdr->{Reason}" if $hdr->{Status} !~ /^2/; - my $data = eval { decode_json $body }; - if(!$data) { - AE::log warn => "$prefix Error decoding JSON: $@"; - return; - } - - my($prod) = $data->{products}->@*; + my $listprice = $body =~ m{<meta property="product:price:amount" content="([^"]+)"} && $1; + my $currency = $body =~ m{<meta property="product:price:currency" content="([^"]+)"} && $1; + my $availability = $body =~ m{<meta property="product:availability" content="([^"]+)"} && $1; + my $sku = $body =~ m{<meta property="product:retailer_item_id" content="([^"]+)"} ? $1 : ''; - if(!$prod || !$prod->{published_at}) { + if(!$listprice || !$availability || $availability ne 'instock') { pg_cmd q{UPDATE shop_denpa SET deadsince = COALESCE(deadsince, NOW()), lastfetch = NOW() WHERE id = $1}, [ $id ]; - AE::log info => "$prefix not found."; + AE::log info => "$prefix not found or not in stock."; } else { - my $price = 'US$ '.$prod->{variants}[0]{price}; - $price = 'free' if $price eq 'US$ 0.00'; + my $price = $listprice eq '0.00' ? 'free' : ($currency eq 'USD' ? 'US$' : $currency).' '.$listprice; pg_cmd 'UPDATE shop_denpa SET deadsince = NULL, lastfetch = NOW(), sku = $2, price = $3 WHERE id = $1', - [ $prod->{handle}, $prod->{variants}[0]{sku}, $price ]; - AE::log debug => "$prefix for $price at $prod->{variants}[0]{sku}"; + [ $id, $sku, $price ]; + AE::log debug => "$prefix for $price at $sku"; } } @@ -73,9 +64,8 @@ sub sync { my $id = $res->value(0,0); my $ts = AE::now; - my $code = encode_base64("$C{user}:$C{pass}", ''); - http_get $C{api}.'?handle='.uri_escape($id), - headers => {'User-Agent' => $C{ua}, Authorization => "Basic $code"}, + http_get sprintf($VNDB::ExtLinks::LINKS{r}{l_denpa}{fmt}, $id), + headers => {'User-Agent' => $C{ua}}, timeout => 60, sub { data(AE::now-$ts, $id, @_) }; }; diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm index f01e8ad8..12c01509 100644 --- a/lib/VNDB/Config.pm +++ b/lib/VNDB/Config.pm @@ -30,7 +30,7 @@ my $config = { trace_log => 0, dlsite_url => 'https://www.dlsite.com/%s/work/=/product_id/%%s.html', - denpa_url => 'https://denpasoft.com/product/%s', + denpa_url => 'https://denpasoft.com/product/%s/', jlist_url => 'https://www.jlist.com/%s', jbox_url => 'https://www.jbox.com/%s', mg_r18_url => 'https://www.mangagamer.com/r18/detail.php?product_code=%d', diff --git a/lib/VNDB/ExtLinks.pm b/lib/VNDB/ExtLinks.pm index e632d9b3..8a28291d 100644 --- a/lib/VNDB/ExtLinks.pm +++ b/lib/VNDB/ExtLinks.pm @@ -97,7 +97,7 @@ our %LINKS = ( , regex => qr{([a-z0-9_-]+\.itch\.io/[a-z0-9_-]+)} , patt => 'https://<artist>.itch.io/<product>' }, l_denpa => { label => 'Denpasoft' - , fmt => 'https://denpasoft.com/product/%s' + , fmt => 'https://denpasoft.com/product/%s/' , fmt2 => config->{denpa_url} , regex => qr{(?:www\.)?denpasoft\.com/products?/([a-z0-9-]+).*} }, l_jlist => { label => 'J-List' |