v2rw/filters: Add experimental validation + move compat to VNWeb

This is a minor start towards rewriting the filter logic in the new v2rw
code. Filters were never validated before (I didn't have a good
framework for it - now I do) and invalid filters would result in a 500.
The new validation code is not yet applied, but failed validations will
be logged so that I can see if it's working correctly.

I mean, what better way to test than to throw it in production?

I'll write new (and more flexible) SQL generation functions for these
filters later, so that v2rw code can get filtered results and I can make
a start on rewriting the pages that depend on the functionality. The
validation schema can also be used to validate the filters stored in the
DB, so I can use it to get rid of the complex handling of stored invalid
filters.

4 files changed, 126 insertions, 17 deletions

diff --git a/lib/VNDB/Func.pm b/lib/VNDB/Func.pm
index 2fcd5b54..508b2272 100644
--- a/lib/VNDB/Func.pm
+++ b/lib/VNDB/Func.pm
@@ -72,7 +72,7 @@ sub fil_serialize {
     my @v = ref $fil->{$_} ? @{$fil->{$_}} : ($fil->{$_});
     s/$e/_$fil_escape{$1}/g for(@v);
     $_.'-'.join '~', @v
-  } grep defined($fil->{$_}), keys %$fil;
+  } grep defined($fil->{$_}), sort keys %$fil;
 }
 
 
diff --git a/lib/VNDB/Util/Misc.pm b/lib/VNDB/Util/Misc.pm
index c08fe1bb..0423e35b 100644
--- a/lib/VNDB/Util/Misc.pm
+++ b/lib/VNDB/Util/Misc.pm
@@ -40,6 +40,8 @@ sub filFetchDB {
 
   my $filters = fil_parse $overwrite // $pref, @{$filfields{$type}};
 
+  VNWeb::Filters::debug_validate($type, $filters);
+
   # compatibility
   my $compat = $self->filCompat($type, $filters);
   $self->authPref($prefname => fil_serialize $filters) if $compat && !defined $overwrite;
@@ -84,22 +86,7 @@ sub filFetchDB {
 # Compatibility with old filters. Modifies the filter in-place and returns the number of changes made.
 sub filCompat {
   my($self, $type, $fil) = @_;
-  my $mod = 0;
-
-  # older tag specification (by name rather than ID)
-  if($type eq 'vn' && ($fil->{taginc} || $fil->{tagexc})) {
-    my $tagfind = sub {
-      return map {
-        my $i = $self->dbTagGet(name => $_)->[0];
-        $i && $i->{searchable} ? $i->{id} : ();
-      } grep $_, ref $_[0] ? @{$_[0]} : ($_[0]||'')
-    };
-    $fil->{tag_inc} //= [ $tagfind->(delete $fil->{taginc}) ] if $fil->{taginc};
-    $fil->{tag_exc} //= [ $tagfind->(delete $fil->{tagexc}) ] if $fil->{tagexc};
-    $mod++;
-  }
-
-  $mod;
+  $type eq 'vn' ? VNWeb::Filters::filter_vn_compat($fil) : 0
 }
 
 
diff --git a/lib/VNWeb/Filters.pm b/lib/VNWeb/Filters.pm
new file mode 100644
index 00000000..8fae51a8
--- /dev/null
+++ b/lib/VNWeb/Filters.pm
@@ -0,0 +1,118 @@
+package VNWeb::Filters;
+
+# This module implements validating and querying the search filters. I'm not
+# sure yet if this filter system will continue to exist in this form or if
+# there will be a better advanced search system to replace it, but either way
+# we'll need to support these filters for the forseeable future.
+
+use VNWeb::Prelude;
+
+my $VN = form_compile any => {
+    date_before => { required => 0, rdate => 1 },
+    date_after  => { required => 0, rdate => 1 },
+    released    => { undefbool => 1 },
+    length      => { required => 0, enum => \%VN_LENGTH },
+    hasani      => { undefbool => 1 },
+    hasshot     => { undefbool => 1 },
+    tag_inc     => { undefarray => { id => 1 } },
+    tag_exc     => { undefarray => { id => 1 } },
+    taginc      => { undefarray => {} }, # [old] Tag search by name
+    tagexc      => { undefarray => {} }, # [old] Tag search by name
+    tagspoil    => { required => 0, default => 0, uint => 1, range => [0,2] },
+    lang        => { undefarray => { enum => \%LANGUAGE } },
+    olang       => { undefarray => { enum => \%LANGUAGE } },
+    plat        => { undefarray => { enum => \%PLATFORM } },
+    staff_inc   => { undefarray => { id => 1 } },
+    staff_exc   => { undefarray => { id => 1 } },
+    ul_notblack => { undefbool => 1 },
+    ul_onwish   => { undefbool => 1 },
+    ul_voted    => { undefbool => 1 },
+    ul_onlist   => { undefbool => 1 },
+};
+
+my $RELEASE = form_compile any => {
+    type        => { required => 0, enum => \%RELEASE_TYPE },
+    patch       => { undefbool => 1 },
+    freeware    => { undefbool => 1 },
+    doujin      => { undefbool => 1 },
+    uncensored  => { undefbool => 1 },
+    date_before => { required => 0, rdate => 1 },
+    date_after  => { required => 0, rdate => 1 },
+    released    => { undefbool => 1 },
+    minage      => { undefarray => { enum => \%AGE_RATING } },
+    lang        => { undefarray => { enum => \%LANGUAGE } },
+    olang       => { undefarray => { enum => \%LANGUAGE } },
+    resolution  => { undefarray => {} },
+    plat        => { undefarray => { enum => \%PLATFORM } },
+    prod_inc    => { undefarray => { id => 1 } },
+    prod_exc    => { undefarray => { id => 1 } },
+    med         => { undefarray => { enum => \%MEDIUM } },
+    voiced      => { undefarray => { enum => \%VOICED } },
+    ani_story   => { undefarray => { enum => \%ANIMATED } },
+    ani_ero     => { undefarray => { enum => \%ANIMATED } },
+    engine      => { required => 0 },
+};
+
+my $CHAR = form_compile any => {
+    gender      => { required => 0, enum => \%GENDER },
+    bloodt      => { required => 0, enum => \%BLOOD_TYPE },
+    bust_min    => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    bust_max    => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    waist_min   => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    waist_max   => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    hip_min     => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    hip_max     => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    height_min  => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    height_max  => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    weight_min  => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    weight_max  => { required => 0, uint => 1, range => [ 0, 32767 ] },
+    cup_min     => { required => 0, enum => \%CUP_SIZE },
+    cup_max     => { required => 0, enum => \%CUP_SIZE },
+    va_inc      => { undefarray => { id => 1 } },
+    va_exc      => { undefarray => { id => 1 } },
+    trait_inc   => { undefarray => { id => 1 } },
+    trait_exc   => { undefarray => { id => 1 } },
+    tagspoil    => { required => 0, default => 0, uint => 1, range => [0,2] },
+    role        => { required => 0, enum => \%CHAR_ROLE },
+};
+
+my $STAFF = form_compile any => {
+    gender      => { required => 0, enum => [qw[unknown m f]] },
+    role        => { undefarray => { enum => [ 'seiyuu', keys %CREDIT_TYPE ] } },
+    truename    => { undefbool => 1 },
+    lang        => { undefarray => { enum => \%LANGUAGE } },
+};
+
+
+sub debug_validate {
+    my($type, $data) = @_;
+    my $s = {vn => $VN, release => $RELEASE, char => $CHAR, staff => $STAFF}->{$type};
+    my $v = $s->validate($data);
+    if(!$v) {
+        warn sprintf "Filter validation failed!\nData: %s\nError: %s", JSON::XS->new->canonical->pretty->encode($data), JSON::XS->new->canonical->pretty->encode($v->err);
+    } else {
+        #warn sprintf "Filter validated: %sSerialized: %s", JSON::XS->new->canonical->pretty->encode($v->data), VNDB::Func::fil_serialize($v->data);
+    }
+}
+
+
+# Compatibility with old VN filters. Modifies the filter in-place and returns the number of changes made.
+sub filter_vn_compat {
+    my($fil) = @_; #XXX: This function is called from old VNDB:: code and the filter data may not have been normalized as per the schema.
+    my $mod = 0;
+
+    # older tag specification (by name rather than ID)
+    for ('taginc', 'tagexc') {
+        my $l = delete $fil->{$_};
+        next if !$l;
+        $l = [ map lc($_), ref $l ? @$l : $l ];
+        $fil->{ s/^tag/tag_/rg } ||= [ map $_->{id}, tuwf->dbAlli(
+           'SELECT DISTINCT id FROM tags LEFT JOIN tags_aliases ON id = tag WHERE searchable AND lower(name) IN', $l, 'OR lower(alias) IN', $l
+        )->@* ];
+        $mod++;
+    }
+
+    $mod;
+}
+
+1;
diff --git a/lib/VNWeb/Validation.pm b/lib/VNWeb/Validation.pm
index d40f42c6..4d398aac 100644
--- a/lib/VNWeb/Validation.pm
+++ b/lib/VNWeb/Validation.pm
@@ -37,6 +37,10 @@ TUWF::set custom_validations => {
     language    => { enum => \%LANGUAGE },
     gtin        => { required => 0, default => 0, func => sub { $_[0] = 0 if !length $_[0]; $_[0] eq 0 || gtintype($_[0]) } },
     rdate       => { uint => 1, func => \&_validate_rdate },
+    # A tri-state bool, returns undef if not present or empty, normalizes to 0/1 otherwise
+    undefbool   => { required => 0, default => undef, func => sub { $_[0] = $_[0] ? 1 : 0; 1 } },
+    # An array that may be either missing (returns undef), a single scalar (returns single-element array) or a proper array
+    undefarray  => sub { +{ required => 0, default => undef, type => 'array', scalar => 1, values => $_[0] } },
     # Accepts a user-entered vote string (or '-' or empty) and converts that into a DB vote number (or undef) - opposite of fmtvote()
     vnvote      => { required => 0, default => undef, regex => qr/^(?:|-|[1-9]|10|[1-9]\.[0-9]|10\.0)$/, func => sub { $_[0] = $_[0] eq '-' ? undef : 10*$_[0]; 1 } },
     # Sort an array by the listed hash keys, using string comparison on each key