summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2020-11-03 13:30:17 +0100
committerYorhel <git@yorhel.nl>2020-11-03 13:30:55 +0100
commit3e81dbd2dd55d32518e756491dedfeccfaa6ce41 (patch)
treebc0bee41dd0ab30ece692e48e828f384bd77f046 /lib
parentf3420b701aa033c90678216bd36d374c10e3ac9f (diff)
Multi::Denpa: Update affiliate crawler to use their new site
Diffstat (limited to 'lib')
-rw-r--r--lib/Multi/Denpa.pm34
-rw-r--r--lib/VNDB/Config.pm2
-rw-r--r--lib/VNDB/ExtLinks.pm2
3 files changed, 14 insertions, 24 deletions
diff --git a/lib/Multi/Denpa.pm b/lib/Multi/Denpa.pm
index bdecd085..6076adc1 100644
--- a/lib/Multi/Denpa.pm
+++ b/lib/Multi/Denpa.pm
@@ -4,16 +4,11 @@ use strict;
use warnings;
use Multi::Core;
use AnyEvent::HTTP;
-use JSON::XS 'decode_json';
-use MIME::Base64 'encode_base64';
use VNDB::Config;
-use TUWF::Misc 'uri_escape';
+use VNDB::ExtLinks ();
my %C = (
- api => '',
- user => '',
- pass => '',
clean_timeout => 48*3600,
check_timeout => 15*60,
);
@@ -44,24 +39,20 @@ sub data {
my $prefix = sprintf '[%.1fs] %s', $time, $id;
return AE::log warn => "$prefix ERROR: $hdr->{Status} $hdr->{Reason}" if $hdr->{Status} !~ /^2/;
- my $data = eval { decode_json $body };
- if(!$data) {
- AE::log warn => "$prefix Error decoding JSON: $@";
- return;
- }
-
- my($prod) = $data->{products}->@*;
+ my $listprice = $body =~ m{<meta property="product:price:amount" content="([^"]+)"} && $1;
+ my $currency = $body =~ m{<meta property="product:price:currency" content="([^"]+)"} && $1;
+ my $availability = $body =~ m{<meta property="product:availability" content="([^"]+)"} && $1;
+ my $sku = $body =~ m{<meta property="product:retailer_item_id" content="([^"]+)"} ? $1 : '';
- if(!$prod || !$prod->{published_at}) {
+ if(!$listprice || !$availability || $availability ne 'instock') {
pg_cmd q{UPDATE shop_denpa SET deadsince = COALESCE(deadsince, NOW()), lastfetch = NOW() WHERE id = $1}, [ $id ];
- AE::log info => "$prefix not found.";
+ AE::log info => "$prefix not found or not in stock.";
} else {
- my $price = 'US$ '.$prod->{variants}[0]{price};
- $price = 'free' if $price eq 'US$ 0.00';
+ my $price = $listprice eq '0.00' ? 'free' : ($currency eq 'USD' ? 'US$' : $currency).' '.$listprice;
pg_cmd 'UPDATE shop_denpa SET deadsince = NULL, lastfetch = NOW(), sku = $2, price = $3 WHERE id = $1',
- [ $prod->{handle}, $prod->{variants}[0]{sku}, $price ];
- AE::log debug => "$prefix for $price at $prod->{variants}[0]{sku}";
+ [ $id, $sku, $price ];
+ AE::log debug => "$prefix for $price at $sku";
}
}
@@ -73,9 +64,8 @@ sub sync {
my $id = $res->value(0,0);
my $ts = AE::now;
- my $code = encode_base64("$C{user}:$C{pass}", '');
- http_get $C{api}.'?handle='.uri_escape($id),
- headers => {'User-Agent' => $C{ua}, Authorization => "Basic $code"},
+ http_get sprintf($VNDB::ExtLinks::LINKS{r}{l_denpa}{fmt}, $id),
+ headers => {'User-Agent' => $C{ua}},
timeout => 60,
sub { data(AE::now-$ts, $id, @_) };
};
diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm
index f01e8ad8..12c01509 100644
--- a/lib/VNDB/Config.pm
+++ b/lib/VNDB/Config.pm
@@ -30,7 +30,7 @@ my $config = {
trace_log => 0,
dlsite_url => 'https://www.dlsite.com/%s/work/=/product_id/%%s.html',
- denpa_url => 'https://denpasoft.com/product/%s',
+ denpa_url => 'https://denpasoft.com/product/%s/',
jlist_url => 'https://www.jlist.com/%s',
jbox_url => 'https://www.jbox.com/%s',
mg_r18_url => 'https://www.mangagamer.com/r18/detail.php?product_code=%d',
diff --git a/lib/VNDB/ExtLinks.pm b/lib/VNDB/ExtLinks.pm
index e632d9b3..8a28291d 100644
--- a/lib/VNDB/ExtLinks.pm
+++ b/lib/VNDB/ExtLinks.pm
@@ -97,7 +97,7 @@ our %LINKS = (
, regex => qr{([a-z0-9_-]+\.itch\.io/[a-z0-9_-]+)}
, patt => 'https://<artist>.itch.io/<product>' },
l_denpa => { label => 'Denpasoft'
- , fmt => 'https://denpasoft.com/product/%s'
+ , fmt => 'https://denpasoft.com/product/%s/'
, fmt2 => config->{denpa_url}
, regex => qr{(?:www\.)?denpasoft\.com/products?/([a-z0-9-]+).*} },
l_jlist => { label => 'J-List'