Add (fullish) database dumps

9 files changed, 1590 insertions, 206 deletions

diff --git a/lib/VNDBSchema.pm b/lib/VNDBSchema.pm
new file mode 100644
index 00000000..5865fe43
--- /dev/null
+++ b/lib/VNDBSchema.pm
@@ -0,0 +1,125 @@
+# Utility functions to parse the files in util/sql/ and extract information and
+# perform a few simple sanity checks.
+#
+# This is not a full-blown SQL parser. The code makes all kinds of assumptions
+# about the formatting of the .sql files.
+
+package VNDBSchema;
+
+use strict;
+use warnings;
+
+# Reads schema.sql and returns a hashref with the following structure:
+# {
+#   vn => {
+#       dbentry_type => 'v',
+#       cols => [
+#           {
+#               name => 'id',
+#               type => 'serial',
+#               decl => 'id SERIAL', # full declaration, exluding comments and PRIMARY KEY marker
+#               pub => 1,
+#           }, ...
+#       ],
+#       primary => ['id'],
+#   }
+# }
+sub schema {
+    my $fn = shift;
+    my %schema;
+    my $table;
+    open my $F, '<', $fn or die "$fn: $!";
+    while(<$F>) {
+        chomp;
+        next if /^\s*--/ || /^\s*$/;
+
+        if(/^\s*CREATE\s+TABLE\s+([^ ]+)/) {
+            die "Unexpected 'CREATE TABLE $1'\n" if $table;
+            $table = $1;
+            $schema{$table}{dbentry_type} = $1 if /--.*\s+dbentry_type=(.)/;
+            $schema{$table}{cols} = [];
+
+        } elsif(/^\s*\);/) {
+            $table = undef;
+
+        } elsif(/^\s+CHECK/) {
+            # ignore
+
+        } elsif($table && /^\s+PRIMARY\s+KEY\s*\(([^\)]+)\)/i) {
+            die "Double primary key for '$table'?\n" if $schema{$table}{primary};
+            $schema{$table}{primary} = [ map s/\s*"?([^\s"]+)"?\s*/$1/r, split /,/, $1 ];
+
+        } elsif($table && s/^\s+"?([^"\( ]+)"?\s+//) {
+            my $col = { name => $1 };
+            push @{$schema{$table}{cols}}, $col;
+
+            $col->{pub} = /--.*\[pub\]/;
+            s/,?\s*(?:--.*)?$//;
+
+            if(s/\s+PRIMARY\s+KEY//i) {
+                die "Double primary key for '$table'?\n" if $schema{$table}{primary};
+                $schema{$table}{primary} = [ $col->{name} ];
+            }
+            $col->{decl} = "\"$col->{name}\" $_";
+            $col->{type} = lc s/^([^ ]+)\s.+/$1/r;
+
+        } else {
+            die "Unrecognized line in schema.sql: $_\n";
+        }
+    }
+
+    \%schema
+}
+
+
+# Parses types from all.sql and returns a hashref with the following structure:
+# {
+#   anime_type => {
+#       decl => 'CREATE TYPE ..;'
+#   }, ..
+# }
+sub types {
+    my $fn = shift;
+    my %types;
+    open my $F, '<', $fn or die "$fn: $!";
+    while(<$F>) {
+        chomp;
+        if(/^CREATE TYPE ([^ ]+)/) {
+            $types{$1} = { decl => $_ };
+        }
+    }
+    \%types
+}
+
+
+# Parses foreign key references from tableattrs.sql and returns an arrayref:
+# [
+#   {
+#       decl => 'ALTER TABLE ..;',
+#       from_table => 'vn_anime',
+#       from_cols => ['id'],
+#       to_table => 'vn',
+#       to_cols => ['id'],
+#       name => 'vn_anime_id_fkey'
+#   }, ..
+# ]
+sub references {
+    my $fn = shift;
+    my @ref;
+    open my $F, '<', $fn or die "$fn: $!";
+    while(<$F>) {
+        chomp;
+        next if !/^\s*ALTER\s+TABLE\s+([^ ]+)\s+ADD\s+CONSTRAINT\s+([^ ]+)\s+FOREIGN\s+KEY\s+\(([^\)]+)\)\s+REFERENCES\s+([^ ]+)\s*\(([^\)]+)\)/;
+        push @ref, {
+            decl => $_,
+            from_table => $1,
+            name => $2,
+            from_cols => [ map s/"//r, split /\s*,\s*/, $3 ],
+            to_table => $4,
+            to_cols => [ map s/"//r, split /\s*,\s*/, $5 ]
+        };
+    }
+    \@ref
+}
+
+1;
diff --git a/util/dbdump.pl b/util/dbdump.pl
new file mode 100755
index 00000000..79075626
--- /dev/null
+++ b/util/dbdump.pl
@@ -0,0 +1,251 @@
+#!/usr/bin/perl
+my $HELP=<<_;
+Usage:
+
+util/dbdump.pl export-db output.tar.zst
+
+  Write a full database export as a .tar.zst
+
+  The uncompressed directory is written to "output.tar.zst_dir"
+
+util/dbdump.pl export-img output.tar.zst
+
+  Write an export of all referenced images to a .tar.zst
+_
+
+# TODO:
+# - Import
+# - Consolidate with devdump.pl?
+
+use strict;
+use warnings;
+use autodie;
+use DBI;
+use DBD::Pg;
+use File::Copy 'cp';
+
+use Cwd 'abs_path';
+our $ROOT;
+BEGIN { ($ROOT = abs_path $0) =~ s{/util/dbdump\.pl$}{}; }
+
+use lib "$ROOT/lib";
+use VNDBSchema;
+
+
+# Tables and columns to export.
+#
+# Tables are exported with an explicit ORDER BY to make them more deterministic
+# and avoid potentially leaking information about internal state (such as when
+# a user last updated their account).
+my %tables = (
+    anime               => { where => 'id IN(SELECT va.aid FROM vn_anime va JOIN vn v ON v.id = va.id WHERE NOT v.hidden)' },
+    chars               => { where => 'NOT hidden' },
+    chars_traits        => { where => 'id IN(SELECT id FROM chars WHERE NOT hidden) AND tid IN(SELECT id FROM traits WHERE state = 2)' },
+    chars_vns           => { where => 'id IN(SELECT id FROM chars WHERE NOT hidden)'
+                                .' AND vid IN(SELECT id FROM vn WHERE NOT hidden)'
+                                .' AND rid IN(SELECT id FROM releases WHERE NOT hidden)'
+                           , order => 'id, vid, rid' },
+    docs                => { where => 'NOT hidden' },
+    producers           => { where => 'NOT hidden' },
+    producers_relations => { where => 'id IN(SELECT id FROM producers WHERE NOT hidden)' },
+    releases            => { where => 'NOT hidden' },
+    releases_lang       => { where => 'id IN(SELECT id FROM releases WHERE NOT hidden)' },
+    releases_media      => { where => 'id IN(SELECT id FROM releases WHERE NOT hidden)' },
+    releases_platforms  => { where => 'id IN(SELECT id FROM releases WHERE NOT hidden)' },
+    releases_producers  => { where => 'id IN(SELECT id FROM releases WHERE NOT hidden) AND pid IN(SELECT id FROM producers WHERE NOT hidden)' },
+    releases_vn         => { where => 'id IN(SELECT id FROM releases WHERE NOT hidden) AND vid IN(SELECT id FROM vn WHERE NOT hidden)' },
+    rlists              => { where => 'uid NOT IN(SELECT uid FROM users_prefs WHERE key = \'hide_list\') AND rid IN(SELECT id FROM releases WHERE NOT hidden)' },
+    screenshots         => { where => 'id IN(SELECT scr FROM vn_screenshots vs JOIN vn v ON v.id = vs.id WHERE NOT v.hidden)' },
+    staff               => { where => 'NOT hidden' },
+    staff_alias         => { where => 'id IN(SELECT id FROM staff WHERE NOT hidden)' },
+    tags                => { where => 'state = 2' },
+    tags_aliases        => { where => 'tag IN(SELECT id FROM tags WHERE state = 2)' },
+    tags_parents        => { where => 'tag IN(SELECT id FROM tags WHERE state = 2)' },
+    tags_vn             => { where => 'tag IN(SELECT id FROM tags WHERE state = 2) AND vid IN(SELECT id FROM vn WHERE NOT hidden)' },
+    traits              => { where => 'state = 2' },
+    traits_parents      => { where => 'trait IN(SELECT id FROM traits WHERE state = 2)' },
+                           # Only include users that are relevant for this dump.
+                           # (The 'DISTINCT' isn't necessary, but does make the query faster)
+                           # (Users with their votes ignored are still included. W/e)
+    users               => { where => q{
+                                (     id NOT IN(SELECT DISTINCT uid FROM users_prefs WHERE key = 'hide_list')
+                                  AND id IN(SELECT DISTINCT uid FROM rlists
+                                      UNION SELECT DISTINCT uid FROM wlists
+                                      UNION SELECT DISTINCT uid FROM vnlists
+                                      UNION SELECT DISTINCT uid FROM votes)
+                                ) OR id IN(SELECT DISTINCT uid FROM tags_vn)
+                           } },
+    vn                  => { where => 'NOT hidden' },
+    vn_anime            => { where => 'id IN(SELECT id FROM vn WHERE NOT hidden)' },
+    vn_relations        => { where => 'id IN(SELECT id FROM vn WHERE NOT hidden)' },
+    vn_screenshots      => { where => 'id IN(SELECT id FROM vn WHERE NOT hidden)' },
+    vn_seiyuu           => { where => 'id IN(SELECT id FROM vn WHERE NOT hidden)'
+                                .' AND aid IN(SELECT sa.aid FROM staff_alias sa JOIN staff s ON s.id = sa.id WHERE NOT s.hidden)'
+                                .' AND cid IN(SELECT id FROM chars WHERE NOT hidden)' },
+    vn_staff            => { where => 'id IN(SELECT id FROM vn WHERE NOT hidden) AND aid IN(SELECT sa.aid FROM staff_alias sa JOIN staff s ON s.id = sa.id WHERE NOT s.hidden)' },
+    vnlists             => { where => 'uid NOT IN(SELECT uid FROM users_prefs WHERE key = \'hide_list\') AND vid IN(SELECT id FROM vn WHERE NOT hidden)' },
+    votes               => { where => 'uid NOT IN(SELECT uid FROM users_prefs WHERE key = \'hide_list\')'
+                                .' AND uid NOT IN(SELECT id FROM users WHERE ign_votes)'
+                                .' AND vid IN(SELECT id FROM vn WHERE NOT hidden)' },
+    wlists              => { where => 'uid NOT IN(SELECT uid FROM users_prefs WHERE key = \'hide_list\') AND vid IN(SELECT id FROM vn WHERE NOT hidden)' },
+);
+
+my @tables = map +{ name => $_, %{$tables{$_}} }, sort keys %tables;
+my $schema = VNDBSchema::schema("$ROOT/util/sql/schema.sql");
+my $types = VNDBSchema::types("$ROOT/util/sql/all.sql");
+my $references = VNDBSchema::references("$ROOT/util/sql/tableattrs.sql");
+
+my $db = DBI->connect('dbi:Pg:dbname=vndb', 'vndb', undef, { RaiseError => 1 });
+$db->do('SET TIME ZONE +0');
+
+
+sub export_timestamp {
+    my $dest = shift;
+    open my $F, '>', $dest;
+    printf $F "%s\n", $db->selectrow_array('SELECT date_trunc(\'second\', NOW())');
+}
+
+
+sub export_table {
+    my($dest, $table) = @_;
+
+    my $schema = $schema->{$table->{name}};
+    my @cols = grep $_->{pub}, @{$schema->{cols}};
+    die "No columns to export for table '$table->{name}'\n" if !@cols;;
+
+    print "# Dumping $table->{name}\n";
+    my $fn = "$dest/$table->{name}";
+
+    # Truncate all timestamptz columns to a day, to avoid leaking privacy-sensitive info.
+    my $cols = join ', ', map $_->{type} eq 'timestamptz' ? "date_trunc('day', \"$_->{name}\")" : qq{"$_->{name}"}, @cols;
+    my $where = $table->{where} ? "WHERE $table->{where}" : '';
+    my $order = $schema->{primary} ? join ', ', map "\"$_\"", @{$schema->{primary}} : $table->{order};
+    die "Table '$table->{name}' is missing an ORDER BY clause\n" if !$order;
+
+    $db->do(qq{COPY (SELECT $cols FROM "$table->{name}" $where ORDER BY $order) TO STDOUT});
+    open my $F, '>:utf8', $fn;
+    my $v;
+    print $F $v while($db->pg_getcopydata($v) >= 0);
+    close $F;
+
+    open $F, '>', "$fn.header";
+    print $F join "\t", map $_->{name}, @cols;
+    print $F "\n";
+    close $F;
+}
+
+
+sub export_import_script {
+    my $dest = shift;
+    open my $F, '>', $dest;
+    print $F <<'    _' =~ s/^    //mgr;
+    -- This script will create the necessary tables and import all data into an
+    -- existing PostgreSQL database.
+    --
+    -- Usage:
+    --   Run a 'CREATE DATABASE $database' somewhere.
+    --   psql -U $user $database -f import.sql
+    --
+    -- The imported database does not include any indices, other than primary keys.
+    -- You may want to create some indices by hand to speed up complex queries.
+
+    -- Uncomment to import the schema and data into a separate namespace:
+    --CREATE SCHEMA vndb;
+    --SET search_path TO vndb;
+    _
+
+    print $F "\n\n";
+    my %types = map +($_->{type}, 1), grep $_->{pub}, map @{$schema->{$_->{name}}{cols}}, @tables;
+    print $F "$types->{$_}{decl}\n" for (sort grep $types->{$_}, keys %types);
+
+    for my $table (@tables) {
+        my $schema = $schema->{$table->{name}};
+        print $F "\n";
+        print $F "CREATE TABLE \"$table->{name}\" (\n";
+        print $F join ",\n", map "  $_->{decl}" =~ s/" serial/" integer/ir, grep $_->{pub}, @{$schema->{cols}};
+        print $F ",\n  PRIMARY KEY(".join(', ', map "\"$_\"", @{$schema->{primary}}).")" if $schema->{primary};
+        print $F "\n);\n";
+    }
+
+    print $F "\n\n";
+    print $F "-- You can comment out tables you don't need, to speed up the import and save some disk space.\n";
+    print $F "\\copy $_->{name} from 'db/$_->{name}'\n" for @tables;
+
+    print $F "\n\n";
+    print $F "-- These are included to verify the internal consistency of the dump, you can safely comment out this part.\n";
+    for my $ref (@$references) {
+        next if !$tables{$ref->{from_table}} || !$tables{$ref->{to_table}};
+        my %pub = map +($_->{name}, 1), grep $_->{pub}, @{$schema->{$ref->{from_table}}{cols}};
+        next if grep !$pub{$_}, @{$ref->{from_cols}};
+        print $F "$ref->{decl}\n";
+    }
+}
+
+
+sub export_db {
+    my $dest = shift;
+
+    my @static = qw{
+        LICENSE-CC-BY-NC-SA.txt
+        LICENSE-DBCL.txt
+        LICENSE-ODBL.txt
+        README.txt
+    };
+
+    # This will die if it already exists, which is good because we want to write to a new empty dir.
+    mkdir "${dest}_dir";
+    mkdir "${dest}_dir/db";
+
+    cp "$ROOT/util/dump/$_", "${dest}_dir/$_" for @static;
+
+    export_timestamp "${dest}_dir/TIMESTAMP";
+    export_table "${dest}_dir/db", $_ for @tables;
+    export_import_script "${dest}_dir/import.sql";
+
+    print "# Compressing\n";
+    `tar -cf "$dest" -I 'zstd -7' --sort=name -C "${dest}_dir" @static TIMESTAMP db`
+}
+
+
+# XXX: This does not include images that are linked from descriptions; May want to borrow from util/unusedimages.pl to find those.
+sub export_img {
+    my $dest = shift;
+
+    mkdir "${dest}_dir";
+    cp "$ROOT/util/dump/LICENSE-ODBL.txt", "${dest}_dir/LICENSE-ODBL.txt";
+    cp "$ROOT/util/dump/README-img.txt", "${dest}_dir/README.txt";
+    export_timestamp "${dest}_dir/TIMESTAMP";
+
+    open my $F, '>', "${dest}_files";
+
+    printf $F "static/sf/%1\$02d/%2\$d.jpg\nstatic/st/%1\$02d/%2\$d.jpg\n", $_->[0]%100, $_->[0]
+        for $db->selectall_array("SELECT id FROM screenshots WHERE $tables{screenshots}{where} ORDER BY id");
+
+    printf $F "static/cv/%02d/%d.jpg\n", $_->[0]%100, $_->[0]
+        for $db->selectall_array("SELECT image FROM vn WHERE image <> 0 AND $tables{vn}{where} ORDER BY image");
+
+    printf $F "static/ch/%02d/%d.jpg\n", $_->[0]%100, $_->[0]
+        for $db->selectall_array("SELECT image FROM chars WHERE image <> 0 AND $tables{chars}{where} ORDER BY image");
+
+    close $F;
+    undef $db;
+
+    `tar -cf "$dest" -I 'zstd -5' \\
+        --verbatim-files-from --files-from "${dest}_files" \\
+        -C "${dest}_dir" LICENSE-ODBL.txt README.txt TIMESTAMP`;
+
+    unlink "${dest}_files";
+    unlink "${dest}_dir/LICENSE-ODBL.txt";
+    unlink "${dest}_dir/README.txt";
+    unlink "${dest}_dir/TIMESTAMP";
+    rmdir "${dest}_dir";
+}
+
+
+if($ARGV[0] && $ARGV[0] eq 'export-db' && $ARGV[1]) {
+    export_db $ARGV[1];
+} elsif($ARGV[0] && $ARGV[0] eq 'export-img' && $ARGV[1]) {
+    export_img $ARGV[1];
+} else {
+    print $HELP;
+}
diff --git a/util/dump/LICENSE-CC-BY-NC-SA.txt b/util/dump/LICENSE-CC-BY-NC-SA.txt
new file mode 100644
index 00000000..5797ceb3
--- /dev/null
+++ b/util/dump/LICENSE-CC-BY-NC-SA.txt
@@ -0,0 +1,362 @@
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Creative
+Commons Corporation ("Creative Commons") is not a law firm and does not provide
+legal services or legal advice. Distribution of Creative Commons public licenses
+does not create a lawyer-client or other relationship. Creative Commons makes
+its licenses and related information available on an "as-is" basis. Creative
+Commons gives no warranties regarding its licenses, any material licensed
+under their terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the fullest
+extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and conditions
+that creators and other rights holders may use to share original works of
+authorship and other material subject to copyright and certain other rights
+specified in the public license below. The following considerations are for
+informational purposes only, are not exhaustive, and do not form part of our
+licenses.
+
+Considerations for licensors: Our public licenses are intended for use by
+those authorized to give the public permission to use material in ways otherwise
+restricted by copyright and certain other rights. Our licenses are irrevocable.
+Licensors should read and understand the terms and conditions of the license
+they choose before applying it. Licensors should also secure all rights necessary
+before applying our licenses so that the public can reuse the material as
+expected. Licensors should clearly mark any material not subject to the license.
+This includes other CC-licensed material, or material used under an exception
+or limitation to copyright. More considerations for licensors : wiki.creativecommons.org/Considerations_for_licensors
+
+Considerations for the public: By using one of our public licenses, a licensor
+grants the public permission to use the licensed material under specified
+terms and conditions. If the licensor's permission is not necessary for any
+reason–for example, because of any applicable exception or limitation to copyright–then
+that use is not regulated by the license. Our licenses grant only permissions
+under copyright and certain other rights that a licensor has authority to
+grant. Use of the licensed material may still be restricted for other reasons,
+including because others have copyright or other rights in the material. A
+licensor may make special requests, such as asking that all changes be marked
+or described. Although not required by our licenses, you are encouraged to
+respect those requests where reasonable. More considerations for the public
+: wiki.creativecommons.org/Considerations_for_licensees
+
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree to
+be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial-ShareAlike
+4.0 International Public License ("Public License"). To the extent this Public
+License may be interpreted as a contract, You are granted the Licensed Rights
+in consideration of Your acceptance of these terms and conditions, and the
+Licensor grants You such rights in consideration of benefits the Licensor
+receives from making the Licensed Material available under these terms and
+conditions.
+
+Section 1 – Definitions.
+
+a. Adapted Material means material subject to Copyright and Similar Rights
+that is derived from or based upon the Licensed Material and in which the
+Licensed Material is translated, altered, arranged, transformed, or otherwise
+modified in a manner requiring permission under the Copyright and Similar
+Rights held by the Licensor. For purposes of this Public License, where the
+Licensed Material is a musical work, performance, or sound recording, Adapted
+Material is always produced where the Licensed Material is synched in timed
+relation with a moving image.
+
+b. Adapter's License means the license You apply to Your Copyright and Similar
+Rights in Your contributions to Adapted Material in accordance with the terms
+and conditions of this Public License.
+
+c. BY-NC-SA Compatible License means a license listed at creativecommons.org/compatiblelicenses,
+approved by Creative Commons as essentially the equivalent of this Public
+License.
+
+d. Copyright and Similar Rights means copyright and/or similar rights closely
+related to copyright including, without limitation, performance, broadcast,
+sound recording, and Sui Generis Database Rights, without regard to how the
+rights are labeled or categorized. For purposes of this Public License, the
+rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights.
+
+e. Effective Technological Measures means those measures that, in the absence
+of proper authority, may not be circumvented under laws fulfilling obligations
+under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996,
+and/or similar international agreements.
+
+f. Exceptions and Limitations means fair use, fair dealing, and/or any other
+exception or limitation to Copyright and Similar Rights that applies to Your
+use of the Licensed Material.
+
+g. License Elements means the license attributes listed in the name of a Creative
+Commons Public License. The License Elements of this Public License are Attribution,
+NonCommercial, and ShareAlike.
+
+h. Licensed Material means the artistic or literary work, database, or other
+material to which the Licensor applied this Public License.
+
+i. Licensed Rights means the rights granted to You subject to the terms and
+conditions of this Public License, which are limited to all Copyright and
+Similar Rights that apply to Your use of the Licensed Material and that the
+Licensor has authority to license.
+
+j. Licensor means the individual(s) or entity(ies) granting rights under this
+Public License.
+
+k. NonCommercial means not primarily intended for or directed towards commercial
+advantage or monetary compensation. For purposes of this Public License, the
+exchange of the Licensed Material for other material subject to Copyright
+and Similar Rights by digital file-sharing or similar means is NonCommercial
+provided there is no payment of monetary compensation in connection with the
+exchange.
+
+l. Share means to provide material to the public by any means or process that
+requires permission under the Licensed Rights, such as reproduction, public
+display, public performance, distribution, dissemination, communication, or
+importation, and to make material available to the public including in ways
+that members of the public may access the material from a place and at a time
+individually chosen by them.
+
+m. Sui Generis Database Rights means rights other than copyright resulting
+from Directive 96/9/EC of the European Parliament and of the Council of 11
+March 1996 on the legal protection of databases, as amended and/or succeeded,
+as well as other essentially equivalent rights anywhere in the world.
+
+n. You means the individual or entity exercising the Licensed Rights under
+this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+   a. License grant.
+
+1. Subject to the terms and conditions of this Public License, the Licensor
+hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive,
+irrevocable license to exercise the Licensed Rights in the Licensed Material
+to:
+
+A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial
+purposes only; and
+
+B. produce, reproduce, and Share Adapted Material for NonCommercial purposes
+only.
+
+2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions
+and Limitations apply to Your use, this Public License does not apply, and
+You do not need to comply with its terms and conditions.
+
+      3. Term. The term of this Public License is specified in Section 6(a).
+
+4. Media and formats; technical modifications allowed. The Licensor authorizes
+You to exercise the Licensed Rights in all media and formats whether now known
+or hereafter created, and to make technical modifications necessary to do
+so. The Licensor waives and/or agrees not to assert any right or authority
+to forbid You from making technical modifications necessary to exercise the
+Licensed Rights, including technical modifications necessary to circumvent
+Effective Technological Measures. For purposes of this Public License, simply
+making modifications authorized by this Section 2(a)(4) never produces Adapted
+Material.
+
+      5. Downstream recipients.
+
+A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed
+Material automatically receives an offer from the Licensor to exercise the
+Licensed Rights under the terms and conditions of this Public License.
+
+B. Additional offer from the Licensor – Adapted Material. Every recipient
+of Adapted Material from You automatically receives an offer from the Licensor
+to exercise the Licensed Rights in the Adapted Material under the conditions
+of the Adapter's License You apply.
+
+C. No downstream restrictions. You may not offer or impose any additional
+or different terms or conditions on, or apply any Effective Technological
+Measures to, the Licensed Material if doing so restricts exercise of the Licensed
+Rights by any recipient of the Licensed Material.
+
+6. No endorsement. Nothing in this Public License constitutes or may be construed
+as permission to assert or imply that You are, or that Your use of the Licensed
+Material is, connected with, or sponsored, endorsed, or granted official status
+by, the Licensor or others designated to receive attribution as provided in
+Section 3(a)(1)(A)(i).
+
+   b. Other rights.
+
+1. Moral rights, such as the right of integrity, are not licensed under this
+Public License, nor are publicity, privacy, and/or other similar personality
+rights; however, to the extent possible, the Licensor waives and/or agrees
+not to assert any such rights held by the Licensor to the limited extent necessary
+to allow You to exercise the Licensed Rights, but not otherwise.
+
+2. Patent and trademark rights are not licensed under this Public License.
+
+3. To the extent possible, the Licensor waives any right to collect royalties
+from You for the exercise of the Licensed Rights, whether directly or through
+a collecting society under any voluntary or waivable statutory or compulsory
+licensing scheme. In all other cases the Licensor expressly reserves any right
+to collect such royalties, including when the Licensed Material is used other
+than for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following
+conditions.
+
+   a. Attribution.
+
+1. If You Share the Licensed Material (including in modified form), You must:
+
+A. retain the following if it is supplied by the Licensor with the Licensed
+Material:
+
+i. identification of the creator(s) of the Licensed Material and any others
+designated to receive attribution, in any reasonable manner requested by the
+Licensor (including by pseudonym if designated);
+
+            ii. a copyright notice;
+
+            iii. a notice that refers to this Public License;
+
+            iv. a notice that refers to the disclaimer of warranties;
+
+            
+
+v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
+
+B. indicate if You modified the Licensed Material and retain an indication
+of any previous modifications; and
+
+C. indicate the Licensed Material is licensed under this Public License, and
+include the text of, or the URI or hyperlink to, this Public License.
+
+2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner
+based on the medium, means, and context in which You Share the Licensed Material.
+For example, it may be reasonable to satisfy the conditions by providing a
+URI or hyperlink to a resource that includes the required information.
+
+3. If requested by the Licensor, You must remove any of the information required
+by Section 3(a)(1)(A) to the extent reasonably practicable.
+
+b. ShareAlike.In addition to the conditions in Section 3(a), if You Share
+Adapted Material You produce, the following conditions also apply.
+
+1. The Adapter's License You apply must be a Creative Commons license with
+the same License Elements, this version or later, or a BY-NC-SA Compatible
+License.
+
+2. You must include the text of, or the URI or hyperlink to, the Adapter's
+License You apply. You may satisfy this condition in any reasonable manner
+based on the medium, means, and context in which You Share Adapted Material.
+
+3. You may not offer or impose any additional or different terms or conditions
+on, or apply any Effective Technological Measures to, Adapted Material that
+restrict exercise of the rights granted under the Adapter's License You apply.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to
+Your use of the Licensed Material:
+
+a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract,
+reuse, reproduce, and Share all or a substantial portion of the contents of
+the database for NonCommercial purposes only;
+
+b. if You include all or a substantial portion of the database contents in
+a database in which You have Sui Generis Database Rights, then the database
+in which You have Sui Generis Database Rights (but not its individual contents)
+is Adapted Material, including for purposes of Section 3(b); and
+
+c. You must comply with the conditions in Section 3(a) if You Share all or
+a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not replace
+Your obligations under this Public License where the Licensed Rights include
+other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+a. Unless otherwise separately undertaken by the Licensor, to the extent possible,
+the Licensor offers the Licensed Material as-is and as-available, and makes
+no representations or warranties of any kind concerning the Licensed Material,
+whether express, implied, statutory, or other. This includes, without limitation,
+warranties of title, merchantability, fitness for a particular purpose, non-infringement,
+absence of latent or other defects, accuracy, or the presence or absence of
+errors, whether or not known or discoverable. Where disclaimers of warranties
+are not allowed in full or in part, this disclaimer may not apply to You.
+
+b. To the extent possible, in no event will the Licensor be liable to You
+on any legal theory (including, without limitation, negligence) or otherwise
+for any direct, special, indirect, incidental, consequential, punitive, exemplary,
+or other losses, costs, expenses, or damages arising out of this Public License
+or use of the Licensed Material, even if the Licensor has been advised of
+the possibility of such losses, costs, expenses, or damages. Where a limitation
+of liability is not allowed in full or in part, this limitation may not apply
+to You.
+
+c. The disclaimer of warranties and limitation of liability provided above
+shall be interpreted in a manner that, to the extent possible, most closely
+approximates an absolute disclaimer and waiver of all liability.
+
+Section 6 – Term and Termination.
+
+a. This Public License applies for the term of the Copyright and Similar Rights
+licensed here. However, if You fail to comply with this Public License, then
+Your rights under this Public License terminate automatically.
+
+b. Where Your right to use the Licensed Material has terminated under Section
+6(a), it reinstates:
+
+1. automatically as of the date the violation is cured, provided it is cured
+within 30 days of Your discovery of the violation; or
+
+      2. upon express reinstatement by the Licensor.
+
+For the avoidance of doubt, this Section 6(b) does not affect any right the
+Licensor may have to seek remedies for Your violations of this Public License.
+
+c. For the avoidance of doubt, the Licensor may also offer the Licensed Material
+under separate terms or conditions or stop distributing the Licensed Material
+at any time; however, doing so will not terminate this Public License.
+
+   d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 – Other Terms and Conditions.
+
+a. The Licensor shall not be bound by any additional or different terms or
+conditions communicated by You unless expressly agreed.
+
+b. Any arrangements, understandings, or agreements regarding the Licensed
+Material not stated herein are separate from and independent of the terms
+and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+a. For the avoidance of doubt, this Public License does not, and shall not
+be interpreted to, reduce, limit, restrict, or impose conditions on any use
+of the Licensed Material that could lawfully be made without permission under
+this Public License.
+
+b. To the extent possible, if any provision of this Public License is deemed
+unenforceable, it shall be automatically reformed to the minimum extent necessary
+to make it enforceable. If the provision cannot be reformed, it shall be severed
+from this Public License without affecting the enforceability of the remaining
+terms and conditions.
+
+c. No term or condition of this Public License will be waived and no failure
+to comply consented to unless expressly agreed to by the Licensor.
+
+d. Nothing in this Public License constitutes or may be interpreted as a limitation
+upon, or waiver of, any privileges and immunities that apply to the Licensor
+or You, including from the legal processes of any jurisdiction or authority.
+
+Creative Commons is not a party to its public licenses. Notwithstanding, Creative
+Commons may elect to apply one of its public licenses to material it publishes
+and in those instances will be considered the "Licensor." The text of the
+Creative Commons public licenses is dedicated to the public domain under the
+CC0 Public Domain Dedication. Except for the limited purpose of indicating
+that material is shared under a Creative Commons public license or as otherwise
+permitted by the Creative Commons policies published at creativecommons.org/policies,
+Creative Commons does not authorize the use of the trademark "Creative Commons"
+or any other trademark or logo of Creative Commons without its prior written
+consent including, without limitation, in connection with any unauthorized
+modifications to any of its public licenses or any other arrangements, understandings,
+or agreements concerning use of licensed material. For the avoidance of doubt,
+this paragraph does not form part of the public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
diff --git a/util/dump/LICENSE-DBCL.txt b/util/dump/LICENSE-DBCL.txt
new file mode 100644
index 00000000..f8239599
--- /dev/null
+++ b/util/dump/LICENSE-DBCL.txt
@@ -0,0 +1,52 @@
+## ODC Database Contents License
+
+The Licensor and You agree as follows: 
+
+### 1.0 Definitions of Capitalised Words
+
+The definitions of the Open Database License (ODbL) 1.0 are incorporated
+by reference into the Database Contents License.
+
+### 2.0 Rights granted and Conditions of Use
+
+2.1 Rights granted. The Licensor grants to You a worldwide,
+royalty-free, non-exclusive, perpetual, irrevocable copyright license to
+do any act that is restricted by copyright over anything within the
+Contents, whether in the original medium or any other. These rights
+explicitly include commercial use, and do not exclude any field of
+endeavour.  These rights include, without limitation, the right to
+sublicense the work.
+
+2.2 Conditions of Use. You must comply with the ODbL.
+
+2.3 Relationship to Databases and ODbL. This license does not cover any
+Database Rights, Database copyright, or contract over the Contents as
+part of the Database. Please see the ODbL covering the Database for more
+details about Your rights and obligations.
+
+2.4 Non-assertion of copyright over facts. The Licensor takes the
+position that factual information is not covered by copyright. The DbCL
+grants you permission for any information having copyright contained in
+the Contents.
+
+### 3.0 Warranties, disclaimer, and limitation of liability
+
+3.1 The Contents are licensed by the Licensor "as is" and without any
+warranty of any kind, either express or implied, whether of title, of
+accuracy, of the presence of absence of errors, of fitness for purpose,
+or otherwise. Some jurisdictions do not allow the exclusion of implied
+warranties, so this exclusion may not apply to You.
+
+3.2 Subject to any liability that may not be excluded or limited by law,
+the Licensor is not liable for, and expressly excludes, all liability
+for loss or damage however and whenever caused to anyone by any use
+under this License, whether by You or by anyone else, and whether caused
+by any fault on the part of the Licensor or not. This exclusion of
+liability includes, but is not limited to, any special, incidental,
+consequential, punitive, or exemplary damages. This exclusion applies
+even if the Licensor has been advised of the possibility of such
+damages.
+
+3.3 If liability may not be excluded by law, it is limited to actual and
+direct financial loss to the extent it is caused by proved negligence on
+the part of the Licensor.
diff --git a/util/dump/LICENSE-ODBL.txt b/util/dump/LICENSE-ODBL.txt
new file mode 100644
index 00000000..21528871
--- /dev/null
+++ b/util/dump/LICENSE-ODBL.txt
@@ -0,0 +1,540 @@
+## ODC Open Database License (ODbL)

+

+### Preamble

+

+The Open Database License (ODbL) is a license agreement intended to

+allow users to freely share, modify, and use this Database while

+maintaining this same freedom for others. Many databases are covered by

+copyright, and therefore this document licenses these rights. Some

+jurisdictions, mainly in the European Union, have specific rights that

+cover databases, and so the ODbL addresses these rights, too. Finally,

+the ODbL is also an agreement in contract for users of this Database to

+act in certain ways in return for accessing this Database.

+

+Databases can contain a wide variety of types of content (images,

+audiovisual material, and sounds all in the same database, for example),

+and so the ODbL only governs the rights over the Database, and not the

+contents of the Database individually. Licensors should use the ODbL

+together with another license for the contents, if the contents have a

+single set of rights that uniformly covers all of the contents. If the

+contents have multiple sets of different rights, Licensors should

+describe what rights govern what contents together in the individual

+record or in some other way that clarifies what rights apply. 

+

+Sometimes the contents of a database, or the database itself, can be

+covered by other rights not addressed here (such as private contracts,

+trade mark over the name, or privacy rights / data protection rights

+over information in the contents), and so you are advised that you may

+have to consult other documents or clear other rights before doing

+activities not covered by this License.

+

+------

+

+The Licensor (as defined below) 

+

+and 

+

+You (as defined below) 

+

+agree as follows: 

+

+### 1.0 Definitions of Capitalised Words

+

+"Collective Database" – Means this Database in unmodified form as part

+of a collection of independent databases in themselves that together are

+assembled into a collective whole. A work that constitutes a Collective

+Database will not be considered a Derivative Database.

+

+"Convey" – As a verb, means Using the Database, a Derivative Database,

+or the Database as part of a Collective Database in any way that enables

+a Person to make or receive copies of the Database or a Derivative

+Database.  Conveying does not include interaction with a user through a

+computer network, or creating and Using a Produced Work, where no

+transfer of a copy of the Database or a Derivative Database occurs.

+"Contents" – The contents of this Database, which includes the

+information, independent works, or other material collected into the

+Database. For example, the contents of the Database could be factual

+data or works such as images, audiovisual material, text, or sounds.

+

+"Database" – A collection of material (the Contents) arranged in a

+systematic or methodical way and individually accessible by electronic

+or other means offered under the terms of this License.

+

+"Database Directive" – Means Directive 96/9/EC of the European

+Parliament and of the Council of 11 March 1996 on the legal protection

+of databases, as amended or succeeded.

+

+"Database Right" – Means rights resulting from the Chapter III ("sui

+generis") rights in the Database Directive (as amended and as transposed

+by member states), which includes the Extraction and Re-utilisation of

+the whole or a Substantial part of the Contents, as well as any similar

+rights available in the relevant jurisdiction under Section 10.4. 

+

+"Derivative Database" – Means a database based upon the Database, and

+includes any translation, adaptation, arrangement, modification, or any

+other alteration of the Database or of a Substantial part of the

+Contents. This includes, but is not limited to, Extracting or

+Re-utilising the whole or a Substantial part of the Contents in a new

+Database.

+

+"Extraction" – Means the permanent or temporary transfer of all or a

+Substantial part of the Contents to another medium by any means or in

+any form.

+

+"License" – Means this license agreement and is both a license of rights

+such as copyright and Database Rights and an agreement in contract.

+

+"Licensor" – Means the Person that offers the Database under the terms

+of this License. 

+

+"Person" – Means a natural or legal person or a body of persons

+corporate or incorporate.

+

+"Produced Work" –  a work (such as an image, audiovisual material, text,

+or sounds) resulting from using the whole or a Substantial part of the

+Contents (via a search or other query) from this Database, a Derivative

+Database, or this Database as part of a Collective Database.  

+

+"Publicly" – means to Persons other than You or under Your control by

+either more than 50% ownership or by the power to direct their

+activities (such as contracting with an independent consultant). 

+

+"Re-utilisation" – means any form of making available to the public all

+or a Substantial part of the Contents by the distribution of copies, by

+renting, by online or other forms of transmission.

+

+"Substantial" – Means substantial in terms of quantity or quality or a

+combination of both. The repeated and systematic Extraction or

+Re-utilisation of insubstantial parts of the Contents may amount to the

+Extraction or Re-utilisation of a Substantial part of the Contents.

+

+"Use" – As a verb, means doing any act that is restricted by copyright

+or Database Rights whether in the original medium or any other; and

+includes without limitation distributing, copying, publicly performing,

+publicly displaying, and preparing derivative works of the Database, as

+well as modifying the Database as may be technically necessary to use it

+in a different mode or format. 

+

+"You" – Means a Person exercising rights under this License who has not

+previously violated the terms of this License with respect to the

+Database, or who has received express permission from the Licensor to

+exercise rights under this License despite a previous violation.

+

+Words in the singular include the plural and vice versa.

+

+### 2.0 What this License covers

+

+2.1. Legal effect of this document. This License is:

+

+  a. A license of applicable copyright and neighbouring rights;

+

+  b. A license of the Database Right; and

+

+  c. An agreement in contract between You and the Licensor.

+

+2.2 Legal rights covered. This License covers the legal rights in the

+Database, including:

+

+  a. Copyright. Any copyright or neighbouring rights in the Database.

+  The copyright licensed includes any individual elements of the

+  Database, but does not cover the copyright over the Contents

+  independent of this Database. See Section 2.4 for details. Copyright

+  law varies between jurisdictions, but is likely to cover: the Database

+  model or schema, which is the structure, arrangement, and organisation

+  of the Database, and can also include the Database tables and table

+  indexes; the data entry and output sheets; and the Field names of

+  Contents stored in the Database;

+

+  b. Database Rights. Database Rights only extend to the Extraction and

+  Re-utilisation of the whole or a Substantial part of the Contents.

+  Database Rights can apply even when there is no copyright over the

+  Database. Database Rights can also apply when the Contents are removed

+  from the Database and are selected and arranged in a way that would

+  not infringe any applicable copyright; and

+

+  c. Contract. This is an agreement between You and the Licensor for

+  access to the Database. In return you agree to certain conditions of

+  use on this access as outlined in this License. 

+

+2.3 Rights not covered. 

+

+  a. This License does not apply to computer programs used in the making

+  or operation of the Database; 

+

+  b. This License does not cover any patents over the Contents or the

+  Database; and

+

+  c. This License does not cover any trademarks associated with the

+  Database. 

+

+2.4 Relationship to Contents in the Database. The individual items of

+the Contents contained in this Database may be covered by other rights,

+including copyright, patent, data protection, privacy, or personality

+rights, and this License does not cover any rights (other than Database

+Rights or in contract) in individual Contents contained in the Database.

+For example, if used on a Database of images (the Contents), this

+License would not apply to copyright over individual images, which could

+have their own separate licenses, or one single license covering all of

+the rights over the images.  

+

+### 3.0 Rights granted

+

+3.1 Subject to the terms and conditions of this License, the Licensor

+grants to You a worldwide, royalty-free, non-exclusive, terminable (but

+only under Section 9) license to Use the Database for the duration of

+any applicable copyright and Database Rights. These rights explicitly

+include commercial use, and do not exclude any field of endeavour. To

+the extent possible in the relevant jurisdiction, these rights may be

+exercised in all media and formats whether now known or created in the

+future. 

+

+The rights granted cover, for example:

+

+  a. Extraction and Re-utilisation of the whole or a Substantial part of

+  the Contents;

+

+  b. Creation of Derivative Databases;

+

+  c. Creation of Collective Databases;

+

+  d. Creation of temporary or permanent reproductions by any means and

+  in any form, in whole or in part, including of any Derivative

+  Databases or as a part of Collective Databases; and

+

+  e. Distribution, communication, display, lending, making available, or

+  performance to the public by any means and in any form, in whole or in

+  part, including of any Derivative Database or as a part of Collective

+  Databases.

+

+3.2 Compulsory license schemes. For the avoidance of doubt:

+

+  a. Non-waivable compulsory license schemes. In those jurisdictions in

+  which the right to collect royalties through any statutory or

+  compulsory licensing scheme cannot be waived, the Licensor reserves

+  the exclusive right to collect such royalties for any exercise by You

+  of the rights granted under this License;

+

+  b. Waivable compulsory license schemes. In those jurisdictions in

+  which the right to collect royalties through any statutory or

+  compulsory licensing scheme can be waived, the Licensor waives the

+  exclusive right to collect such royalties for any exercise by You of

+  the rights granted under this License; and,

+

+  c. Voluntary license schemes. The Licensor waives the right to collect

+  royalties, whether individually or, in the event that the Licensor is

+  a member of a collecting society that administers voluntary licensing

+  schemes, via that society, from any exercise by You of the rights

+  granted under this License.

+

+3.3 The right to release the Database under different terms, or to stop

+distributing or making available the Database, is reserved. Note that

+this Database may be multiple-licensed, and so You may have the choice

+of using alternative licenses for this Database. Subject to Section

+10.4, all other rights not expressly granted by Licensor are reserved.

+

+### 4.0 Conditions of Use

+

+4.1 The rights granted in Section 3 above are expressly made subject to

+Your complying with the following conditions of use. These are important

+conditions of this License, and if You fail to follow them, You will be

+in material breach of its terms.

+

+4.2 Notices. If You Publicly Convey this Database, any Derivative

+Database, or the Database as part of a Collective Database, then You

+must: 

+

+  a. Do so only under the terms of this License or another license

+  permitted under Section 4.4;

+

+  b. Include a copy of this License (or, as applicable, a license

+  permitted under Section 4.4) or its Uniform Resource Identifier (URI)

+  with the Database or Derivative Database, including both in the

+  Database or Derivative Database and in any relevant documentation; and

+

+  c. Keep intact any copyright or Database Right notices and notices

+  that refer to this License.

+

+  d. If it is not possible to put the required notices in a particular

+  file due to its structure, then You must include the notices in a

+  location (such as a relevant directory) where users would be likely to

+  look for it.

+

+4.3 Notice for using output (Contents). Creating and Using a Produced

+Work does not require the notice in Section 4.2. However, if you

+Publicly Use a Produced Work, You must include a notice associated with

+the Produced Work reasonably calculated to make any Person that uses,

+views, accesses, interacts with, or is otherwise exposed to the Produced

+Work aware that Content was obtained from the Database, Derivative

+Database, or the Database as part of a Collective Database, and that it

+is available under this License.

+

+  a. Example notice. The following text will satisfy notice under

+  Section 4.3:

+

+        Contains information from DATABASE NAME, which is made available

+        here under the Open Database License (ODbL).

+

+DATABASE NAME should be replaced with the name of the Database and a

+hyperlink to the URI of the Database. "Open Database License" should

+contain a hyperlink to the URI of the text of this License. If

+hyperlinks are not possible, You should include the plain text of the

+required URI's with the above notice.

+ 

+4.4 Share alike. 

+

+  a. Any Derivative Database that You Publicly Use must be only under

+  the terms of: 

+

+    i. This License;

+

+    ii. A later version of this License similar in spirit to this

+      License; or

+

+    iii. A compatible license. 

+

+  If You license the Derivative Database under one of the licenses

+  mentioned in (iii), You must comply with the terms of that license. 

+

+  b. For the avoidance of doubt, Extraction or Re-utilisation of the

+  whole or a Substantial part of the Contents into a new database is a

+  Derivative Database and must comply with Section 4.4. 

+

+  c. Derivative Databases and Produced Works.  A Derivative Database is

+  Publicly Used and so must comply with Section 4.4. if a Produced Work

+  created from the Derivative Database is Publicly Used.

+

+  d. Share Alike and additional Contents. For the avoidance of doubt,

+  You must not add Contents to Derivative Databases under Section 4.4 a

+  that are incompatible with the rights granted under this License. 

+

+  e. Compatible licenses. Licensors may authorise a proxy to determine

+  compatible licenses under Section 4.4 a iii. If they do so, the

+  authorised proxy's public statement of acceptance of a compatible

+  license grants You permission to use the compatible license.

+

+

+4.5 Limits of Share Alike.  The requirements of Section 4.4 do not apply

+in the following:

+

+  a. For the avoidance of doubt, You are not required to license

+  Collective Databases under this License if You incorporate this

+  Database or a Derivative Database in the collection, but this License

+  still applies to this Database or a Derivative Database as a part of

+  the Collective Database; 

+

+  b. Using this Database, a Derivative Database, or this Database as

+  part of a Collective Database to create a Produced Work does not

+  create a Derivative Database for purposes of  Section 4.4; and

+

+  c. Use of a Derivative Database internally within an organisation is

+  not to the public and therefore does not fall under the requirements

+  of Section 4.4.

+

+4.6 Access to Derivative Databases. If You Publicly Use a Derivative

+Database or a Produced Work from a Derivative Database, You must also

+offer to recipients of the Derivative Database or Produced Work a copy

+in a machine readable form of:

+

+  a. The entire Derivative Database; or

+

+  b. A file containing all of the alterations made to the Database or

+  the method of making the alterations to the Database (such as an

+  algorithm), including any additional Contents, that make up all the

+  differences between the Database and the Derivative Database.

+

+The Derivative Database (under a.) or alteration file (under b.) must be

+available at no more than a reasonable production cost for physical

+distributions and free of charge if distributed over the internet.

+

+4.7 Technological measures and additional terms

+

+  a. This License does not allow You to impose (except subject to

+  Section 4.7 b.)  any terms or any technological measures on the

+  Database, a Derivative Database, or the whole or a Substantial part of

+  the Contents that alter or restrict the terms of this License, or any

+  rights granted under it, or have the effect or intent of restricting

+  the ability of any person to exercise those rights.

+

+  b. Parallel distribution. You may impose terms or technological

+  measures on the Database, a Derivative Database, or the whole or a

+  Substantial part of the Contents (a "Restricted Database") in

+  contravention of Section 4.74 a. only if You also make a copy of the

+  Database or a Derivative Database available to the recipient of the

+  Restricted Database:

+

+    i. That is available without additional fee;

+

+    ii. That is available in a medium that does not alter or restrict

+    the terms of this License, or any rights granted under it, or have

+    the effect or intent of restricting the ability of any person to

+    exercise those rights (an "Unrestricted Database"); and

+

+    iii. The Unrestricted Database is at least as accessible to the

+    recipient as a practical matter as the Restricted Database.

+

+  c. For the avoidance of doubt, You may place this Database or a

+  Derivative Database in an authenticated environment, behind a

+  password, or within a similar access control scheme provided that You

+  do not alter or restrict the terms of this License or any rights

+  granted under it or have the effect or intent of restricting the

+  ability of any person to exercise those rights. 

+

+4.8 Licensing of others. You may not sublicense the Database. Each time

+You communicate the Database, the whole or Substantial part of the

+Contents, or any Derivative Database to anyone else in any way, the

+Licensor offers to the recipient a license to the Database on the same

+terms and conditions as this License. You are not responsible for

+enforcing compliance by third parties with this License, but You may

+enforce any rights that You have over a Derivative Database. You are

+solely responsible for any modifications of a Derivative Database made

+by You or another Person at Your direction. You may not impose any

+further restrictions on the exercise of the rights granted or affirmed

+under this License.

+

+### 5.0 Moral rights

+

+5.1 Moral rights. This section covers moral rights, including any rights

+to be identified as the author of the Database or to object to treatment

+that would otherwise prejudice the author's honour and reputation, or

+any other derogatory treatment:

+

+  a. For jurisdictions allowing waiver of moral rights, Licensor waives

+  all moral rights that Licensor may have in the Database to the fullest

+  extent possible by the law of the relevant jurisdiction under Section

+  10.4; 

+

+  b. If waiver of moral rights under Section 5.1 a in the relevant

+  jurisdiction is not possible, Licensor agrees not to assert any moral

+  rights over the Database and waives all claims in moral rights to the

+  fullest extent possible by the law of the relevant jurisdiction under

+  Section 10.4; and

+

+  c. For jurisdictions not allowing waiver or an agreement not to assert

+  moral rights under Section 5.1 a and b, the author may retain their

+  moral rights over certain aspects of the Database.

+

+Please note that some jurisdictions do not allow for the waiver of moral

+rights, and so moral rights may still subsist over the Database in some

+jurisdictions.

+

+### 6.0 Fair dealing, Database exceptions, and other rights not affected 

+

+6.1 This License does not affect any rights that You or anyone else may

+independently have under any applicable law to make any use of this

+Database, including without limitation:

+

+  a. Exceptions to the Database Right including: Extraction of Contents

+  from non-electronic Databases for private purposes, Extraction for

+  purposes of illustration for teaching or scientific research, and

+  Extraction or Re-utilisation for public security or an administrative

+  or judicial procedure. 

+

+  b. Fair dealing, fair use, or any other legally recognised limitation

+  or exception to infringement of copyright or other applicable laws. 

+

+6.2 This License does not affect any rights of lawful users to Extract

+and Re-utilise insubstantial parts of the Contents, evaluated

+quantitatively or qualitatively, for any purposes whatsoever, including

+creating a Derivative Database (subject to other rights over the

+Contents, see Section 2.4). The repeated and systematic Extraction or

+Re-utilisation of insubstantial parts of the Contents may however amount

+to the Extraction or Re-utilisation of a Substantial part of the

+Contents.

+

+### 7.0 Warranties and Disclaimer

+

+7.1 The Database is licensed by the Licensor "as is" and without any

+warranty of any kind, either express, implied, or arising by statute,

+custom, course of dealing, or trade usage. Licensor specifically

+disclaims any and all implied warranties or conditions of title,

+non-infringement, accuracy or completeness, the presence or absence of

+errors, fitness for a particular purpose, merchantability, or otherwise.

+Some jurisdictions do not allow the exclusion of implied warranties, so

+this exclusion may not apply to You.

+

+### 8.0 Limitation of liability

+

+8.1 Subject to any liability that may not be excluded or limited by law,

+the Licensor is not liable for, and expressly excludes, all liability

+for loss or damage however and whenever caused to anyone by any use

+under this License, whether by You or by anyone else, and whether caused

+by any fault on the part of the Licensor or not. This exclusion of

+liability includes, but is not limited to, any special, incidental,

+consequential, punitive, or exemplary damages such as loss of revenue,

+data, anticipated profits, and lost business. This exclusion applies

+even if the Licensor has been advised of the possibility of such

+damages.

+

+8.2 If liability may not be excluded by law, it is limited to actual and

+direct financial loss to the extent it is caused by proved negligence on

+the part of the Licensor.

+

+### 9.0 Termination of Your rights under this License

+

+9.1 Any breach by You of the terms and conditions of this License

+automatically terminates this License with immediate effect and without

+notice to You. For the avoidance of doubt, Persons who have received the

+Database, the whole or a Substantial part of the Contents, Derivative

+Databases, or the Database as part of a Collective Database from You

+under this License will not have their licenses terminated provided

+their use is in full compliance with this License or a license granted

+under Section 4.8 of this License.  Sections 1, 2, 7, 8, 9 and 10 will

+survive any termination of this License.

+

+9.2 If You are not in breach of the terms of this License, the Licensor

+will not terminate Your rights under it. 

+

+9.3 Unless terminated under Section 9.1, this License is granted to You

+for the duration of applicable rights in the Database. 

+

+9.4 Reinstatement of rights. If you cease any breach of the terms and

+conditions of this License, then your full rights under this License

+will be reinstated:

+

+  a. Provisionally and subject to permanent termination until the 60th

+  day after cessation of breach; 

+

+  b. Permanently on the 60th day after cessation of breach unless

+  otherwise reasonably notified by the Licensor; or

+

+  c.  Permanently if reasonably notified by the Licensor of the

+  violation, this is the first time You have received notice of

+  violation of this License from  the Licensor, and You cure the

+  violation prior to 30 days after your receipt of the notice.

+

+Persons subject to permanent termination of rights are not eligible to

+be a recipient and receive a license under Section 4.8.

+

+9.5 Notwithstanding the above, Licensor reserves the right to release

+the Database under different license terms or to stop distributing or

+making available the Database. Releasing the Database under different

+license terms or stopping the distribution of the Database will not

+withdraw this License (or any other license that has been, or is

+required to be, granted under the terms of this License), and this

+License will continue in full force and effect unless terminated as

+stated above.

+

+### 10.0 General

+

+10.1 If any provision of this License is held to be invalid or

+unenforceable, that must not affect the validity or enforceability of

+the remainder of the terms and conditions of this License and each

+remaining provision of this License shall be valid and enforced to the

+fullest extent permitted by law. 

+

+10.2 This License is the entire agreement between the parties with

+respect to the rights granted here over the Database. It replaces any

+earlier understandings, agreements or representations with respect to

+the Database. 

+

+10.3 If You are in breach of the terms of this License, You will not be

+entitled to rely on the terms of this License or to complain of any

+breach by the Licensor. 

+

+10.4 Choice of law. This License takes effect in and will be governed by

+the laws of the relevant jurisdiction in which the License terms are

+sought to be enforced. If the standard suite of rights granted under

+applicable copyright law and Database Rights in the relevant

+jurisdiction includes additional rights not granted under this License,

+these additional rights are granted in this License in order to meet the

+terms of this License.

diff --git a/util/dump/README-img.txt b/util/dump/README-img.txt
new file mode 100644
index 00000000..498c226d
--- /dev/null
+++ b/util/dump/README-img.txt
@@ -0,0 +1,13 @@
+This is an export of all images referenced from the VNDB.org database.
+See https://vndb.org/d14 for more information.
+
+
+License
+=======
+
+This database is made available under the Open Database License [ODbL].
+
+Images in this database are gathered from various online sources and may be
+subject to separate license conditions.
+
+[ODbL]: LICENSE-ODBL.txt; http://opendatacommons.org/licenses/odbl/1.0/
diff --git a/util/dump/README.txt b/util/dump/README.txt
new file mode 100644
index 00000000..dec74115
--- /dev/null
+++ b/util/dump/README.txt
@@ -0,0 +1,49 @@
+This is an export of the VNDB.org database.
+See https://vndb.org/d14 for more information.
+
+
+Usage
+=====
+
+Each table is exported as a separate file in PostgreSQL's COPY format. This
+format is pretty easy to parse - it's TSV with some escape sequences.
+
+Alternatively, a script is provided to load the data into a PostgreSQL
+database for easy querying. See import.sql for options and usage information.
+
+Note that the included database schema is *not* compatible with the VNDB
+source code, Importing the data into a compatible schema should be possible,
+but this will require some additional scripting.
+
+
+Privacy
+=======
+
+This database dump contains user-contributed information, including personal
+visual novel lists and votes. Users have the option to have their data
+excluded from this dump, but that option is obviously unable to retroactively
+delete data from older dumps. If you republish any data contained in this
+dump, please make sure to synchronise regularly and remove data that is not
+present anymore.
+
+
+License
+=======
+
+This database is made available under the Open Database License [ODbL].
+Any rights in individual contents of the database are licensed under the
+Database Contents License [DbCL].
+
+With the following exceptions:
+
+Anime data (db/anime) is obtained from the AniDB.net UDP API and is licensed
+under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0
+[CC-BY-NC-SA].
+
+Visual novel descriptions (db/vn, 'desc' column) and character descriptions
+(db/chars, 'desc' column) are gathered from various online sources and may be
+subject to separate license conditions.
+
+[ODbL]: LICENSE-ODBL.txt; http://opendatacommons.org/licenses/odbl/1.0/
+[DbCL]: LICENSE-DBCL.txt; http://opendatacommons.org/licenses/dbcl/1.0/
+[CC-BY-NC-SA]: LICENSE-CC-BY-NC-SA.txt; https://creativecommons.org/licenses/by-nc-sa/4.0/
diff --git a/util/sql/schema.sql b/util/sql/schema.sql
index 6ef2cd3b..b62b33a8 100644
--- a/util/sql/schema.sql
+++ b/util/sql/schema.sql
@@ -32,10 +32,15 @@
 -- The changes and *_hist tables contain all the data. In a sense, the other
 -- tables related to the item are just a cache/view into the latest versions.
 -- All modifications to the item tables has to go through the edit_* functions
--- in func.sql, these are also responsible for keeping things synchronized.
+-- in editfunc.sql, these are also responsible for keeping things synchronized.
+--
+-- Columns marked with a '[pub]' comment on the same line are included in the
+-- public database dump. Be aware that not all properties of the to-be-dumped
+-- data is annotated in this file. Which tables and which rows are exported is
+-- defined in util/dbdump.pl.
 --
 -- Note: Every CREATE TABLE clause and each column should be on a separate
--- line. This file is parsed by util/sqleditfunc.pl, and it doesn't implement a
+-- line. This file is parsed by lib/VNDBSchema.pm and it doesn't implement a
 -- full SQL query parser.
 
 
@@ -55,13 +60,13 @@ CREATE TABLE affiliate_links (
 
 -- anime
 CREATE TABLE anime (
-  id integer NOT NULL PRIMARY KEY,
-  year smallint,
-  ann_id integer,
-  nfo_id varchar(200),
-  type anime_type,
-  title_romaji varchar(250),
-  title_kanji varchar(250),
+  id integer NOT NULL PRIMARY KEY, -- [pub]
+  year smallint, -- [pub]
+  ann_id integer, -- [pub]
+  nfo_id varchar(200), -- [pub]
+  type anime_type, -- [pub]
+  title_romaji varchar(250) -- [pub]
+  title_kanji varchar(250), -- [pub]
   lastfetch timestamptz
 );
 
@@ -81,25 +86,25 @@ CREATE TABLE changes (
 
 -- chars
 CREATE TABLE chars ( -- dbentry_type=c
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  name       varchar(250) NOT NULL DEFAULT '',
-  original   varchar(250) NOT NULL DEFAULT '',
-  alias      varchar(500) NOT NULL DEFAULT '',
-  image      integer  NOT NULL DEFAULT 0,
-  "desc"     text     NOT NULL DEFAULT '',
-  gender     gender NOT NULL DEFAULT 'unknown',
-  s_bust     smallint NOT NULL DEFAULT 0,
-  s_waist    smallint NOT NULL DEFAULT 0,
-  s_hip      smallint NOT NULL DEFAULT 0,
-  b_month    smallint NOT NULL DEFAULT 0,
-  b_day      smallint NOT NULL DEFAULT 0,
-  height     smallint NOT NULL DEFAULT 0,
-  weight     smallint,
-  bloodt     blood_type NOT NULL DEFAULT 'unknown',
-  main       integer, -- chars.id
-  main_spoil smallint NOT NULL DEFAULT 0
+  name       varchar(250) NOT NULL DEFAULT '', -- [pub]
+  original   varchar(250) NOT NULL DEFAULT '', -- [pub]
+  alias      varchar(500) NOT NULL DEFAULT '', -- [pub]
+  image      integer  NOT NULL DEFAULT 0, -- [pub]
+  "desc"     text     NOT NULL DEFAULT '', -- [pub]
+  gender     gender NOT NULL DEFAULT 'unknown', -- [pub]
+  s_bust     smallint NOT NULL DEFAULT 0, -- [pub]
+  s_waist    smallint NOT NULL DEFAULT 0, -- [pub]
+  s_hip      smallint NOT NULL DEFAULT 0, -- [pub]
+  b_month    smallint NOT NULL DEFAULT 0, -- [pub]
+  b_day      smallint NOT NULL DEFAULT 0, -- [pub]
+  height     smallint NOT NULL DEFAULT 0, -- [pub]
+  weight     smallint, -- [pub]
+  bloodt     blood_type NOT NULL DEFAULT 'unknown', -- [pub]
+  main       integer, -- [pub] chars.id
+  main_spoil smallint NOT NULL DEFAULT 0 -- [pub]
 );
 
 -- chars_hist
@@ -125,9 +130,9 @@ CREATE TABLE chars_hist (
 
 -- chars_traits
 CREATE TABLE chars_traits (
-  id         integer NOT NULL,
-  tid        integer NOT NULL, -- traits.id
-  spoil      smallint NOT NULL DEFAULT 0,
+  id         integer NOT NULL, -- [pub]
+  tid        integer NOT NULL, -- [pub] traits.id
+  spoil      smallint NOT NULL DEFAULT 0, -- [pub]
   PRIMARY KEY(id, tid)
 );
 
@@ -141,11 +146,11 @@ CREATE TABLE chars_traits_hist (
 
 -- chars_vns
 CREATE TABLE chars_vns (
-  id         integer NOT NULL,
-  vid        integer NOT NULL, -- vn.id
-  rid        integer NULL, -- releases.id
-  spoil      smallint NOT NULL DEFAULT 0,
-  role       char_role NOT NULL DEFAULT 'main'
+  id         integer NOT NULL, -- [pub]
+  vid        integer NOT NULL, -- [pub] vn.id
+  rid        integer NULL, -- [pub] releases.id
+  spoil      smallint NOT NULL DEFAULT 0, -- [pub]
+  role       char_role NOT NULL DEFAULT 'main' -- [pub]
 );
 
 -- chars_vns_hist
@@ -159,11 +164,11 @@ CREATE TABLE chars_vns_hist (
 
 -- docs
 CREATE TABLE docs ( -- dbentry_type=d
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  title      varchar(200) NOT NULL DEFAULT '',
-  content    text NOT NULL DEFAULT ''
+  title      varchar(200) NOT NULL DEFAULT '', -- [pub]
+  content    text NOT NULL DEFAULT '' -- [pub]
 );
 
 -- docs_hist
@@ -195,17 +200,17 @@ CREATE TABLE notifications (
 
 -- producers
 CREATE TABLE producers ( -- dbentry_type=p
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  type       producer_type NOT NULL DEFAULT 'co',
-  name       varchar(200) NOT NULL DEFAULT '',
-  original   varchar(200) NOT NULL DEFAULT '',
-  website    varchar(250) NOT NULL DEFAULT '',
-  lang       language NOT NULL DEFAULT 'ja',
-  "desc"     text NOT NULL DEFAULT '',
-  alias      varchar(500) NOT NULL DEFAULT '',
-  l_wp       varchar(150),
+  type       producer_type NOT NULL DEFAULT 'co', -- [pub]
+  name       varchar(200) NOT NULL DEFAULT '', -- [pub]
+  original   varchar(200) NOT NULL DEFAULT '', -- [pub]
+  website    varchar(250) NOT NULL DEFAULT '', -- [pub]
+  lang       language NOT NULL DEFAULT 'ja', -- [pub]
+  "desc"     text NOT NULL DEFAULT '', -- [pub]
+  alias      varchar(500) NOT NULL DEFAULT '', -- [pub]
+  l_wp       varchar(150), -- [pub]
   rgraph     integer -- relgraphs.id
 );
 
@@ -224,9 +229,9 @@ CREATE TABLE producers_hist (
 
 -- producers_relations
 CREATE TABLE producers_relations (
-  id         integer NOT NULL,
-  pid        integer NOT NULL, -- producers.id
-  relation   producer_relation NOT NULL,
+  id         integer NOT NULL, -- [pub]
+  pid        integer NOT NULL, -- [pub] producers.id
+  relation   producer_relation NOT NULL, -- [pub]
   PRIMARY KEY(id, pid)
 );
 
@@ -247,27 +252,27 @@ CREATE TABLE quotes (
 
 -- releases
 CREATE TABLE releases ( -- dbentry_type=r
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  title      varchar(250) NOT NULL DEFAULT '',
-  original   varchar(250) NOT NULL DEFAULT '',
-  type       release_type NOT NULL DEFAULT 'complete',
-  website    varchar(250) NOT NULL DEFAULT '',
-  catalog    varchar(50) NOT NULL DEFAULT '',
-  gtin       bigint NOT NULL DEFAULT 0,
-  released   integer NOT NULL DEFAULT 0,
-  notes      text NOT NULL DEFAULT '',
-  minage     smallint,
-  patch      boolean NOT NULL DEFAULT FALSE,
-  freeware   boolean NOT NULL DEFAULT FALSE,
-  doujin     boolean NOT NULL DEFAULT FALSE,
-  resolution resolution NOT NULL DEFAULT 'unknown',
-  voiced     smallint NOT NULL DEFAULT 0,
-  ani_story  smallint NOT NULL DEFAULT 0,
-  ani_ero    smallint NOT NULL DEFAULT 0,
-  uncensored boolean NOT NULL DEFAULT FALSE,
-  engine     varchar(50) NOT NULL DEFAULT ''
+  title      varchar(250) NOT NULL DEFAULT '', -- [pub]
+  original   varchar(250) NOT NULL DEFAULT '', -- [pub]
+  type       release_type NOT NULL DEFAULT 'complete', -- [pub]
+  website    varchar(250) NOT NULL DEFAULT '', -- [pub]
+  catalog    varchar(50) NOT NULL DEFAULT '', -- [pub]
+  gtin       bigint NOT NULL DEFAULT 0, -- [pub]
+  released   integer NOT NULL DEFAULT 0, -- [pub]
+  notes      text NOT NULL DEFAULT '', -- [pub]
+  minage     smallint, -- [pub]
+  patch      boolean NOT NULL DEFAULT FALSE, -- [pub]
+  freeware   boolean NOT NULL DEFAULT FALSE, -- [pub]
+  doujin     boolean NOT NULL DEFAULT FALSE, -- [pub]
+  resolution resolution NOT NULL DEFAULT 'unknown', -- [pub]
+  voiced     smallint NOT NULL DEFAULT 0, -- [pub]
+  ani_story  smallint NOT NULL DEFAULT 0, -- [pub]
+  ani_ero    smallint NOT NULL DEFAULT 0, -- [pub]
+  uncensored boolean NOT NULL DEFAULT FALSE, -- [pub]
+  engine     varchar(50) NOT NULL DEFAULT '' -- [pub]
 );
 
 -- releases_hist
@@ -295,8 +300,8 @@ CREATE TABLE releases_hist (
 
 -- releases_lang
 CREATE TABLE releases_lang (
-  id         integer NOT NULL,
-  lang       language NOT NULL,
+  id         integer NOT NULL, -- [pub]
+  lang       language NOT NULL, -- [pub]
   PRIMARY KEY(id, lang)
 );
 
@@ -309,9 +314,9 @@ CREATE TABLE releases_lang_hist (
 
 -- releases_media
 CREATE TABLE releases_media (
-  id         integer NOT NULL,
-  medium     medium NOT NULL,
-  qty        smallint NOT NULL DEFAULT 1,
+  id         integer NOT NULL, -- [pub]
+  medium     medium NOT NULL, -- [pub]
+  qty        smallint NOT NULL DEFAULT 1, -- [pub]
   PRIMARY KEY(id, medium, qty)
 );
 
@@ -325,8 +330,8 @@ CREATE TABLE releases_media_hist (
 
 -- releases_platforms
 CREATE TABLE releases_platforms (
-  id         integer NOT NULL,
-  platform   platform NOT NULL,
+  id         integer NOT NULL, -- [pub]
+  platform   platform NOT NULL, -- [pub]
   PRIMARY KEY(id, platform)
 );
 
@@ -339,10 +344,10 @@ CREATE TABLE releases_platforms_hist (
 
 -- releases_producers
 CREATE TABLE releases_producers (
-  id         integer NOT NULL,
-  pid        integer NOT NULL, -- producers.id
-  developer  boolean NOT NULL DEFAULT FALSE,
-  publisher  boolean NOT NULL DEFAULT TRUE,
+  id         integer NOT NULL, -- [pub]
+  pid        integer NOT NULL, -- [pub] producers.id
+  developer  boolean NOT NULL DEFAULT FALSE, -- [pub]
+  publisher  boolean NOT NULL DEFAULT TRUE, -- [pub]
   CHECK(developer OR publisher),
   PRIMARY KEY(id, pid)
 );
@@ -359,8 +364,8 @@ CREATE TABLE releases_producers_hist (
 
 -- releases_vn
 CREATE TABLE releases_vn (
-  id         integer NOT NULL,
-  vid        integer NOT NULL, -- vn.id
+  id         integer NOT NULL, -- [pub]
+  vid        integer NOT NULL, -- [pub] vn.id
   PRIMARY KEY(id, vid)
 );
 
@@ -379,18 +384,18 @@ CREATE TABLE relgraphs (
 
 -- rlists
 CREATE TABLE rlists (
-  uid integer NOT NULL DEFAULT 0,
-  rid integer NOT NULL DEFAULT 0,
-  status smallint NOT NULL DEFAULT 0,
-  added timestamptz NOT NULL DEFAULT NOW(),
+  uid integer NOT NULL DEFAULT 0, -- [pub]
+  rid integer NOT NULL DEFAULT 0, -- [pub]
+  status smallint NOT NULL DEFAULT 0, -- [pub]
+  added timestamptz NOT NULL DEFAULT NOW(), -- [pub]
   PRIMARY KEY(uid, rid)
 );
 
 -- screenshots
 CREATE TABLE screenshots (
-  id SERIAL NOT NULL PRIMARY KEY,
-  width smallint NOT NULL DEFAULT 0,
-  height smallint NOT NULL DEFAULT 0
+  id SERIAL NOT NULL PRIMARY KEY, -- [pub]
+  width smallint NOT NULL DEFAULT 0, -- [pub]
+  height smallint NOT NULL DEFAULT 0 -- [pub]
 );
 
 -- sessions
@@ -404,17 +409,17 @@ CREATE TABLE sessions (
 
 -- staff
 CREATE TABLE staff ( -- dbentry_type=s
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  aid        integer NOT NULL DEFAULT 0, -- staff_alias.aid
-  gender     gender NOT NULL DEFAULT 'unknown',
-  lang       language NOT NULL DEFAULT 'ja',
-  "desc"     text NOT NULL DEFAULT '',
-  l_wp       varchar(150) NOT NULL DEFAULT '',
-  l_site     varchar(250) NOT NULL DEFAULT '',
-  l_twitter  varchar(16) NOT NULL DEFAULT '',
-  l_anidb    integer
+  aid        integer NOT NULL DEFAULT 0, -- [pub] staff_alias.aid
+  gender     gender NOT NULL DEFAULT 'unknown', -- [pub]
+  lang       language NOT NULL DEFAULT 'ja', -- [pub]
+  "desc"     text NOT NULL DEFAULT '', -- [pub]
+  l_wp       varchar(150) NOT NULL DEFAULT '', -- [pub]
+  l_site     varchar(250) NOT NULL DEFAULT '', -- [pub]
+  l_twitter  varchar(16) NOT NULL DEFAULT '', -- [pub]
+  l_anidb    integer -- [pub]
 );
 
 -- staff_hist
@@ -432,10 +437,10 @@ CREATE TABLE staff_hist (
 
 -- staff_alias
 CREATE TABLE staff_alias (
-  id         integer NOT NULL,
-  aid        SERIAL PRIMARY KEY, -- Globally unique ID of this alias
-  name       varchar(200) NOT NULL DEFAULT '',
-  original   varchar(200) NOT NULL DEFAULT ''
+  id         integer NOT NULL, -- [pub]
+  aid        SERIAL PRIMARY KEY, -- [pub] Globally unique ID of this alias
+  name       varchar(200) NOT NULL DEFAULT '', -- [pub]
+  original   varchar(200) NOT NULL DEFAULT '' -- [pub]
 );
 
 -- staff_alias_hist
@@ -455,41 +460,41 @@ CREATE TABLE stats_cache (
 
 -- tags
 CREATE TABLE tags (
-  id SERIAL NOT NULL PRIMARY KEY,
-  name varchar(250) NOT NULL UNIQUE,
-  description text NOT NULL DEFAULT '',
+  id SERIAL NOT NULL PRIMARY KEY, -- [pub]
+  name varchar(250) NOT NULL UNIQUE, -- [pub]
+  description text NOT NULL DEFAULT '', -- [pub]
   added timestamptz NOT NULL DEFAULT NOW(),
-  state smallint NOT NULL DEFAULT 0,
+  state smallint NOT NULL DEFAULT 0, -- [pub]
   c_items integer NOT NULL DEFAULT 0,
   addedby integer NOT NULL DEFAULT 0,
-  cat tag_category NOT NULL DEFAULT 'cont',
-  defaultspoil smallint NOT NULL DEFAULT 0,
-  searchable boolean NOT NULL DEFAULT TRUE,
-  applicable boolean NOT NULL DEFAULT TRUE
+  cat tag_category NOT NULL DEFAULT 'cont', -- [pub]
+  defaultspoil smallint NOT NULL DEFAULT 0, -- [pub]
+  searchable boolean NOT NULL DEFAULT TRUE, -- [pub]
+  applicable boolean NOT NULL DEFAULT TRUE -- [pub]
 );
 
 -- tags_aliases
 CREATE TABLE tags_aliases (
-  alias varchar(250) NOT NULL PRIMARY KEY,
-  tag integer NOT NULL
+  alias varchar(250) NOT NULL PRIMARY KEY, -- [pub]
+  tag integer NOT NULL -- [pub]
 );
 
 -- tags_parents
 CREATE TABLE tags_parents (
-  tag integer NOT NULL,
-  parent integer NOT NULL,
+  tag integer NOT NULL, -- [pub]
+  parent integer NOT NULL, -- [pub]
   PRIMARY KEY(tag, parent)
 );
 
 -- tags_vn
 CREATE TABLE tags_vn (
-  tag integer NOT NULL,
-  vid integer NOT NULL,
-  uid integer NOT NULL,
-  vote smallint NOT NULL DEFAULT 3 CHECK (vote >= -3 AND vote <= 3 AND vote <> 0),
-  spoiler smallint CHECK(spoiler >= 0 AND spoiler <= 2),
-  date timestamptz NOT NULL DEFAULT NOW(),
-  ignore boolean NOT NULL DEFAULT false,
+  tag integer NOT NULL, -- [pub]
+  vid integer NOT NULL, -- [pub]
+  uid integer NOT NULL, -- [pub]
+  vote smallint NOT NULL DEFAULT 3 CHECK (vote >= -3 AND vote <= 3 AND vote <> 0), -- [pub]
+  spoiler smallint CHECK(spoiler >= 0 AND spoiler <= 2), -- [pub]
+  date timestamptz NOT NULL DEFAULT NOW(), -- [pub]
+  ignore boolean NOT NULL DEFAULT false, -- [pub]
   PRIMARY KEY(tag, vid, uid)
 );
 
@@ -553,20 +558,20 @@ CREATE TABLE threads_boards (
 
 -- traits
 CREATE TABLE traits (
-  id SERIAL PRIMARY KEY,
-  name varchar(250) NOT NULL,
-  alias varchar(500) NOT NULL DEFAULT '',
-  description text NOT NULL DEFAULT '',
+  id SERIAL PRIMARY KEY, -- [pub]
+  name varchar(250) NOT NULL, -- [pub]
+  alias varchar(500) NOT NULL DEFAULT '', -- [pub]
+  description text NOT NULL DEFAULT '', -- [pub]
   added timestamptz NOT NULL DEFAULT NOW(),
-  state smallint NOT NULL DEFAULT 0,
+  state smallint NOT NULL DEFAULT 0, -- [pub]
   addedby integer NOT NULL DEFAULT 0,
-  "group" integer,
-  "order" smallint NOT NULL DEFAULT 0,
-  sexual boolean NOT NULL DEFAULT false,
+  "group" integer, -- [pub]
+  "order" smallint NOT NULL DEFAULT 0, -- [pub]
+  sexual boolean NOT NULL DEFAULT false, -- [pub]
   c_items integer NOT NULL DEFAULT 0,
-  defaultspoil smallint NOT NULL DEFAULT 0,
-  searchable boolean NOT NULL DEFAULT true,
-  applicable boolean NOT NULL DEFAULT true
+  defaultspoil smallint NOT NULL DEFAULT 0, -- [pub]
+  searchable boolean NOT NULL DEFAULT true, -- [pub]
+  applicable boolean NOT NULL DEFAULT true -- [pub]
 );
 
 -- traits_chars
@@ -582,15 +587,15 @@ CREATE TABLE traits_chars (
 
 -- traits_parents
 CREATE TABLE traits_parents (
-  trait integer NOT NULL,
-  parent integer NOT NULL,
+  trait integer NOT NULL, -- [pub]
+  parent integer NOT NULL, -- [pub]
   PRIMARY KEY(trait, parent)
 );
 
 -- users
 CREATE TABLE users (
-  id SERIAL NOT NULL PRIMARY KEY,
-  username varchar(20) NOT NULL UNIQUE,
+  id SERIAL NOT NULL PRIMARY KEY, -- [pub]
+  username varchar(20) NOT NULL UNIQUE, -- [pub]
   mail varchar(100) NOT NULL,
   perm smallint NOT NULL DEFAULT 1+4+16,
   -- Interpretation of the passwd column depends on its length:
@@ -622,19 +627,19 @@ CREATE TABLE users_prefs (
 
 -- vn
 CREATE TABLE vn ( -- dbentry_type=v
-  id         SERIAL PRIMARY KEY,
+  id         SERIAL PRIMARY KEY, -- [pub]
   locked     boolean NOT NULL DEFAULT FALSE,
   hidden     boolean NOT NULL DEFAULT FALSE,
-  title      varchar(250) NOT NULL DEFAULT '',
-  original   varchar(250) NOT NULL DEFAULT '',
-  alias      varchar(500) NOT NULL DEFAULT '',
-  length     smallint NOT NULL DEFAULT 0,
-  img_nsfw   boolean NOT NULL DEFAULT FALSE,
-  image      integer NOT NULL DEFAULT 0,
-  "desc"     text NOT NULL DEFAULT '',
-  l_wp       varchar(150) NOT NULL DEFAULT '',
-  l_encubed  varchar(100) NOT NULL DEFAULT '',
-  l_renai    varchar(100) NOT NULL DEFAULT '',
+  title      varchar(250) NOT NULL DEFAULT '', -- [pub]
+  original   varchar(250) NOT NULL DEFAULT '', -- [pub]
+  alias      varchar(500) NOT NULL DEFAULT '', -- [pub]
+  length     smallint NOT NULL DEFAULT 0, -- [pub]
+  img_nsfw   boolean NOT NULL DEFAULT FALSE, -- [pub]
+  image      integer NOT NULL DEFAULT 0, -- [pub]
+  "desc"     text NOT NULL DEFAULT '', -- [pub]
+  l_wp       varchar(150) NOT NULL DEFAULT '', -- [pub]
+  l_encubed  varchar(100) NOT NULL DEFAULT '', -- [pub]
+  l_renai    varchar(100) NOT NULL DEFAULT '', -- [pub]
   rgraph     integer, -- relgraphs.id
   c_released integer NOT NULL DEFAULT 0,
   c_languages language[] NOT NULL DEFAULT '{}',
@@ -663,8 +668,8 @@ CREATE TABLE vn_hist (
 
 -- vn_anime
 CREATE TABLE vn_anime (
-  id         integer NOT NULL,
-  aid        integer NOT NULL, -- anime.id
+  id         integer NOT NULL, -- [pub]
+  aid        integer NOT NULL, -- [pub] anime.id
   PRIMARY KEY(id, aid)
 );
 
@@ -677,10 +682,10 @@ CREATE TABLE vn_anime_hist (
 
 -- vn_relations
 CREATE TABLE vn_relations (
-  id         integer NOT NULL,
-  vid        integer NOT NULL, -- vn.id
-  relation   vn_relation NOT NULL,
-  official   boolean NOT NULL DEFAULT TRUE,
+  id         integer NOT NULL, -- [pub]
+  vid        integer NOT NULL, -- [pub] vn.id
+  relation   vn_relation NOT NULL, -- [pub]
+  official   boolean NOT NULL DEFAULT TRUE, -- [pub]
   PRIMARY KEY(id, vid)
 );
 
@@ -695,10 +700,10 @@ CREATE TABLE vn_relations_hist (
 
 -- vn_screenshots
 CREATE TABLE vn_screenshots (
-  id         integer NOT NULL,
-  scr        integer NOT NULL, -- screenshots.id
-  rid        integer,          -- releases.id (only NULL for old revisions, nowadays not allowed anymore)
-  nsfw       boolean NOT NULL DEFAULT FALSE,
+  id         integer NOT NULL, -- [pub]
+  scr        integer NOT NULL, -- [pub] screenshots.id
+  rid        integer,          -- [pub] releases.id (only NULL for old revisions, nowadays not allowed anymore)
+  nsfw       boolean NOT NULL DEFAULT FALSE, -- [pub]
   PRIMARY KEY(id, scr)
 );
 
@@ -713,10 +718,10 @@ CREATE TABLE vn_screenshots_hist (
 
 -- vn_seiyuu
 CREATE TABLE vn_seiyuu (
-  id         integer NOT NULL,
-  aid        integer NOT NULL, -- staff_alias.aid
-  cid        integer NOT NULL, -- chars.id
-  note       varchar(250) NOT NULL DEFAULT '',
+  id         integer NOT NULL, -- [pub]
+  aid        integer NOT NULL, -- [pub] staff_alias.aid
+  cid        integer NOT NULL, -- [pub] chars.id
+  note       varchar(250) NOT NULL DEFAULT '', -- [pub]
   PRIMARY KEY (id, aid, cid)
 );
 
@@ -731,10 +736,10 @@ CREATE TABLE vn_seiyuu_hist (
 
 -- vn_staff
 CREATE TABLE vn_staff (
-  id         integer NOT NULL,
-  aid        integer NOT NULL, -- staff_alias.aid
-  role       credit_type NOT NULL DEFAULT 'staff',
-  note       varchar(250) NOT NULL DEFAULT '',
+  id         integer NOT NULL, -- [pub]
+  aid        integer NOT NULL, -- [pub] staff_alias.aid
+  role       credit_type NOT NULL DEFAULT 'staff', -- [pub]
+  note       varchar(250) NOT NULL DEFAULT '', -- [pub]
   PRIMARY KEY (id, aid, role)
 );
 
@@ -749,28 +754,28 @@ CREATE TABLE vn_staff_hist (
 
 -- vnlists
 CREATE TABLE vnlists (
-  uid integer NOT NULL,
-  vid integer NOT NULL,
-  status smallint NOT NULL DEFAULT 0,
-  added TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-  notes varchar NOT NULL DEFAULT '',
+  uid integer NOT NULL, -- [pub]
+  vid integer NOT NULL, -- [pub]
+  status smallint NOT NULL DEFAULT 0, -- [pub]
+  added TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- [pub]
+  notes varchar NOT NULL DEFAULT '', -- [pub]
   PRIMARY KEY(uid, vid)
 );
 
 -- votes
 CREATE TABLE votes (
-  vid integer NOT NULL DEFAULT 0,
-  uid integer NOT NULL DEFAULT 0,
-  vote integer NOT NULL DEFAULT 0,
-  date timestamptz NOT NULL DEFAULT NOW(),
+  vid integer NOT NULL DEFAULT 0, -- [pub]
+  uid integer NOT NULL DEFAULT 0, -- [pub]
+  vote integer NOT NULL DEFAULT 0, -- [pub]
+  date timestamptz NOT NULL DEFAULT NOW(), -- [pub]
   PRIMARY KEY(vid, uid)
 );
 
 -- wlists
 CREATE TABLE wlists (
-  uid integer NOT NULL DEFAULT 0,
-  vid integer NOT NULL DEFAULT 0,
-  wstat smallint NOT NULL DEFAULT 0,
-  added timestamptz NOT NULL DEFAULT NOW(),
+  uid integer NOT NULL DEFAULT 0, -- [pub]
+  vid integer NOT NULL DEFAULT 0, -- [pub]
+  wstat smallint NOT NULL DEFAULT 0, -- [pub]
+  added timestamptz NOT NULL DEFAULT NOW(), -- [pub]
   PRIMARY KEY(uid, vid)
 );
diff --git a/util/sqleditfunc.pl b/util/sqleditfunc.pl
index 773214f9..0d1749a2 100755
--- a/util/sqleditfunc.pl
+++ b/util/sqleditfunc.pl
@@ -8,66 +8,53 @@ use Cwd 'abs_path';
 our $ROOT;
 BEGIN { ($ROOT = abs_path $0) =~ s{/util/sqleditfunc\.pl$}{}; }
 
-my %tabletypes; # table_name => dbentry_type
-my %tables; # table_name => [ column_names ]
-my %items; # item_name => { tables_without_hist => [ data_column_names ] }
-
-
-# Fills %tables
-sub readschema {
-  open my $F, '<', "$ROOT/util/sql/schema.sql" or die $!;
-  my $table = '';
-  while(<$F>) {
-    chomp;
-    if(/^\s*CREATE\s+TABLE\s+([^ ]+)/) {
-      $table = $1;
-      $tables{$table} = [];
-      $tabletypes{$table} = $1 if /--.*\s+dbentry_type=(.)/;
-    } elsif($table && /^\s+("?[^\( ]+"?)\s/ && !/^\s+PRIMARY\s+KEY/) {
-      push @{$tables{$table}}, $1;
-    }
-  }
-}
+use lib "$ROOT/lib";
+use VNDBSchema;
 
+my $schema = VNDBSchema::schema("$ROOT/util/sql/schema.sql");
+my $template = join '', <DATA>;
 
 sub gensql {
-  my($template, $item) = @_;
+  my $item = shift;
 
   # table_name_without_hist => [ column_names_without_chid ]
-  my %ts = map +($_, [ grep !/^chid$/, @{$tables{"${_}_hist"}} ]), map /^${item}_/ && /^(.+)_hist$/ ? $1 : (), keys %tables;
+  my %ts = map
+    +($_, [ map "\"$_->{name}\"", grep $_->{name} !~ /^chid$/, @{$schema->{"${_}_hist"}{cols}} ]),
+    map /^${item}_/ && /^(.+)_hist$/ ? $1 : (), keys %$schema;
+
+  my %replace = ( item => $item, itemtype => $schema->{$item}{dbentry_type} );
 
-  my %replace = ( item => $item, itemtype => $tabletypes{$item} );
   $replace{createtemptables} = join "\n", map sprintf(
     "    CREATE TEMPORARY TABLE edit_%s (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS);\n".
     "    ALTER TABLE edit_%1\$s DROP COLUMN %s;",
     $_, $_ eq 'staff_alias' ? ($_, 'id') : ("${_}_hist", 'chid') # staff_alias copies from the non-_hist table, because it needs the sequence
   ), sort keys %ts;
+
   $replace{temptablenames} = join ', ', map "edit_$_", sort keys %ts;
+
   $replace{loadtemptables} = join "\n", map sprintf(
     "    INSERT INTO edit_%s (%s) SELECT %2\$s FROM %1\$s_hist WHERE chid = xchid;",
     $_, join ', ', @{$ts{$_}}), sort keys %ts;
+
   $replace{copyfromtemp} = join "\n", map sprintf(
     "  DELETE FROM %1\$s WHERE id = r.itemid;\n".
     "  INSERT INTO %1\$s (id, %2\$s) SELECT r.itemid, %2\$s FROM edit_%1\$s;\n".
     "  INSERT INTO %1\$s_hist (chid, %2\$s) SELECT r.chid, %2\$s FROM edit_%1\$s;",
     $_, join ', ', @{$ts{$_}}), grep $_ ne $item, sort keys %ts;
+
   $replace{copymainfromtemp} = sprintf
     "  INSERT INTO %1\$s_hist (chid, %2\$s) SELECT r.chid, %2\$s FROM edit_%1\$s;\n".
     "  UPDATE %1\$s SET locked = (SELECT ilock FROM edit_revision), hidden = (SELECT ihid FROM edit_revision),\n".
     "    %3\$s FROM edit_%1\$s x WHERE id = r.itemid;",
     $item, join(', ', @{$ts{$item}}), join(', ', map "$_ = x.$_", @{$ts{$item}});
 
-  $template =~ s/{([a-z]+)}/$replace{$1}/g;
-  $template;
+  $template =~ s/{([a-z]+)}/$replace{$1}/gr;
 }
 
 
-readschema;
-my $template = join '', <DATA>;
-
 open my $F, '>', "$ROOT/util/sql/editfunc.sql" or die $!;
 print $F "-- Automatically generated by util/sqleditfunc.pl. DO NOT EDIT.\n";
-print $F gensql $template, $_ for sort keys %tabletypes;
+print $F gensql $_ for sort grep $schema->{$_}{dbentry_type}, keys %$schema;
 
 
 __DATA__