summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2019-12-19 13:35:56 +0100
committerYorhel <git@yorhel.nl>2019-12-19 13:35:59 +0100
commit52ed55d9ee83b5dfdb3c9fc7d4f607b232c9447b (patch)
tree7ccb4d3a3d9df1e54e4ead3c2b95cee0a9119eb7 /lib
parent829eb1f2472936d70a70839e55b16858dbcfb6af (diff)
Multi::Denpa: Incrementally fetch shop info by handle lookup
Rather than fetching the entire shop inventory as a single JSON, which doesn't work anymore now that Denpasoft has more products than its API allows you to fetch. Pagination support is kinda meh. The handle lookup method works fine and is also what we do with other shops.
Diffstat (limited to 'lib')
-rw-r--r--lib/Multi/Denpa.pm82
1 files changed, 39 insertions, 43 deletions
diff --git a/lib/Multi/Denpa.pm b/lib/Multi/Denpa.pm
index ac729dce..bdecd085 100644
--- a/lib/Multi/Denpa.pm
+++ b/lib/Multi/Denpa.pm
@@ -7,13 +7,15 @@ use AnyEvent::HTTP;
use JSON::XS 'decode_json';
use MIME::Base64 'encode_base64';
use VNDB::Config;
+use TUWF::Misc 'uri_escape';
my %C = (
api => '',
user => '',
pass => '',
- check_timeout => 24*3600,
+ clean_timeout => 48*3600,
+ check_timeout => 15*60,
);
@@ -22,67 +24,61 @@ sub run {
$C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version};
%C = (%C, @_);
+ push_watcher schedule 0, $C{clean_timeout}, sub {
+ pg_cmd 'DELETE FROM shop_denpa WHERE id NOT IN(SELECT l_denpa FROM releases WHERE NOT hidden)';
+ };
push_watcher schedule 0, $C{check_timeout}, sub {
- pg_cmd 'DELETE FROM shop_denpa WHERE id NOT IN(SELECT l_denpa FROM releases WHERE NOT hidden)', [], sub {
- pg_cmd q{
- INSERT INTO shop_denpa (id)
- SELECT DISTINCT l_denpa
- FROM releases
- WHERE NOT hidden AND l_denpa <> ''
- AND NOT EXISTS(SELECT 1 FROM shop_denpa WHERE id = l_denpa)
- }, [], \&sync
- }
+ pg_cmd q{
+ INSERT INTO shop_denpa (id)
+ SELECT DISTINCT l_denpa
+ FROM releases
+ WHERE NOT hidden AND l_denpa <> ''
+ AND NOT EXISTS(SELECT 1 FROM shop_denpa WHERE id = l_denpa)
+ }, [], \&sync
}
}
-sub sync {
- pg_cmd 'SELECT id FROM shop_denpa', [],
- sub {
- my($res, $time) = @_;
- return if pg_expect $res, 1 or !$res->nRows;
-
- my %handles = map +($res->value($_,0), 1), 0..($res->nRows-1);
-
- my $code = encode_base64("$C{user}:$C{pass}", '');
- http_get $C{api},
- headers => {'User-Agent' => $C{ua}, Authorization => "Basic $code"},
- timeout => 60,
- sub { data(\%handles, @_) };
- };
-}
-
-
sub data {
- my($handles, $body, $hdr) = @_;
+ my($time, $id, $body, $hdr) = @_;
+ my $prefix = sprintf '[%.1fs] %s', $time, $id;
+ return AE::log warn => "$prefix ERROR: $hdr->{Status} $hdr->{Reason}" if $hdr->{Status} !~ /^2/;
- return AE::log warn => "ERROR: $hdr->{Status} $hdr->{Reason}" if $hdr->{Status} !~ /^2/;
my $data = eval { decode_json $body };
if(!$data) {
- AE::log warn => "Error decoding JSON: $@";
+ AE::log warn => "$prefix Error decoding JSON: $@";
return;
}
- AE::log warn => 'Close to result limit, may need to add pagination support' if @{$data->{products}} >= 240;
- my $db_count = keys %$handles;
+ my($prod) = $data->{products}->@*;
- for my $prod (@{$data->{products}}) {
- next if !$prod->{published_at};
+ if(!$prod || !$prod->{published_at}) {
+ pg_cmd q{UPDATE shop_denpa SET deadsince = COALESCE(deadsince, NOW()), lastfetch = NOW() WHERE id = $1}, [ $id ];
+ AE::log info => "$prefix not found.";
- if(!$handles->{$prod->{handle}}) {
- AE::log info => 'Handle not in vndb: https://denpasoft.com/products/%s', $prod->{handle};
- next;
- }
+ } else {
my $price = 'US$ '.$prod->{variants}[0]{price};
$price = 'free' if $price eq 'US$ 0.00';
pg_cmd 'UPDATE shop_denpa SET deadsince = NULL, lastfetch = NOW(), sku = $2, price = $3 WHERE id = $1',
[ $prod->{handle}, $prod->{variants}[0]{sku}, $price ];
-
- delete $handles->{$prod->{handle}};
+ AE::log debug => "$prefix for $price at $prod->{variants}[0]{sku}";
}
+}
- pg_cmd 'UPDATE shop_denpa SET deadsince = COALESCE(deadsince, NOW()), lastfetch = NOW() WHERE id = $1', [$_]
- for (keys %$handles);
- AE::log info => "%d in shop, %d online, %d offline", scalar @{$data->{products}}, $db_count-scalar keys %$handles, scalar keys %$handles;
+sub sync {
+ pg_cmd 'SELECT id FROM shop_denpa ORDER BY lastfetch ASC NULLS FIRST LIMIT 1', [], sub {
+ my($res, $time) = @_;
+ return if pg_expect $res, 1 or !$res->nRows;
+
+ my $id = $res->value(0,0);
+ my $ts = AE::now;
+ my $code = encode_base64("$C{user}:$C{pass}", '');
+ http_get $C{api}.'?handle='.uri_escape($id),
+ headers => {'User-Agent' => $C{ua}, Authorization => "Basic $code"},
+ timeout => 60,
+ sub { data(AE::now-$ts, $id, @_) };
+ };
}
+
+1;