diff options
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | Makefile | 13 | ||||
-rw-r--r-- | README.md | 5 | ||||
-rw-r--r-- | data/conf_example.pl | 41 | ||||
-rw-r--r-- | data/config3_example.pl | 26 | ||||
-rw-r--r-- | data/config_example.pl | 41 | ||||
-rw-r--r-- | data/global.pl | 107 | ||||
-rw-r--r-- | lib/Multi/API.pm | 17 | ||||
-rw-r--r-- | lib/Multi/Core.pm | 21 | ||||
-rw-r--r-- | lib/Multi/DLsite.pm | 3 | ||||
-rw-r--r-- | lib/Multi/Denpa.pm | 3 | ||||
-rw-r--r-- | lib/Multi/Feed.pm | 35 | ||||
-rw-r--r-- | lib/Multi/IRC.pm | 18 | ||||
-rw-r--r-- | lib/Multi/JList.pm | 3 | ||||
-rw-r--r-- | lib/Multi/MG.pm | 3 | ||||
-rw-r--r-- | lib/Multi/Maintenance.pm | 9 | ||||
-rw-r--r-- | lib/Multi/PlayAsia.pm | 3 | ||||
-rw-r--r-- | lib/Multi/Wikidata.pm | 3 | ||||
-rw-r--r-- | lib/VN3/Auth.pm | 6 | ||||
-rw-r--r-- | lib/VNDB/Config.pm | 86 | ||||
-rw-r--r-- | lib/VNDB/Handler/Misc.pm | 2 | ||||
-rw-r--r-- | lib/VNDB/Util/Auth.pm | 2 | ||||
-rw-r--r-- | lib/VNDB/Util/LayoutHTML.pm | 9 | ||||
-rwxr-xr-x | util/jsgen.pl | 8 | ||||
-rwxr-xr-x | util/multi.pl | 17 | ||||
-rwxr-xr-x | util/vndb-dev-server.pl | 4 | ||||
-rwxr-xr-x | util/vndb.pl | 39 | ||||
-rwxr-xr-x | util/vndb3.pl | 7 |
28 files changed, 233 insertions, 301 deletions
@@ -1,5 +1,4 @@ -/data/config.pl -/data/config3.pl +/data/conf.pl /data/docker-pg /data/icons/icons.css /data/log/ @@ -25,7 +25,7 @@ ALL_KEEP=\ static/ch static/cv static/sf static/st \ data/log static/f static/v3 www www/feeds www/api \ - data/config.pl data/config3.pl \ + data/conf.pl \ www/robots.txt static/robots.txt ALL_CLEAN=\ @@ -67,11 +67,8 @@ static/ch static/cv static/sf static/st: data/log www www/feeds www/api static/f static/v3: mkdir -p $@ -data/config.pl: - cp -n data/config_example.pl data/config.pl - -data/config3.pl: - cp -n data/config3_example.pl data/config3.pl +data/conf.pl: + cp -n data/conf_example.pl data/conf.pl %/robots.txt: | www echo 'User-agent: *' > $@ @@ -80,7 +77,7 @@ data/config3.pl: %.gz: % zopfli $< -static/f/vndb.js: data/js/*.js lib/VNDB/Types.pm util/jsgen.pl data/config.pl data/global.pl | static/f +static/f/vndb.js: data/js/*.js lib/VNDB/Types.pm util/jsgen.pl data/conf.pl | static/f util/jsgen.pl static/f/vndb.min.js: static/f/vndb.js @@ -99,7 +96,7 @@ static/s/%/style.css: static/s/%/conf util/skingen.pl data/style.css data/icons/ static/s/%/style.min.css: static/s/%/style.css perl -MCSS::Minifier::XS -e 'undef $$/; print CSS::Minifier::XS::minify(scalar <>)' <$< >$@ -elm3/Lib/Gen.elm: lib/VN3/*.pm lib/VN3/*/*.pm data/config3.pl +elm3/Lib/Gen.elm: lib/VN3/*.pm lib/VN3/*/*.pm data/conf.pl util/vndb3.pl elmgen >$@ static/v3/elm.js: elm3/*.elm elm3/*/*.elm elm3/Lib/Gen.elm | static/f @@ -107,8 +107,8 @@ util/multi.pl (application server, optional): rm dump.sql ``` -- Update `data/config.pl` and optionally `config3.pl` with the proper - credentials for *vndb_site* and *vndb_multi*. +- Update `data/conf.pl` with the proper credentials for *vndb_site* and + *vndb_multi*. - Now simply run: ``` @@ -132,7 +132,6 @@ code is easy to identify, the following files are only used by version 3: - `css3/` - `elm3/` - `util/vndb3.pl` -- `data/config3{,_example}.pl` To run version 3 instead of 2: diff --git a/data/conf_example.pl b/data/conf_example.pl new file mode 100644 index 00000000..5885e8e9 --- /dev/null +++ b/data/conf_example.pl @@ -0,0 +1,41 @@ +{ + # Canonical URL of this site + url => 'http://localhost:3000', + # And of the static files (leave unset to use `url`) + #url_static => 'http://localhost:3000', + + # Salt used to generate the CSRF tokens + form_salt => '<some unique string>', + # Global salt used to hash user passwords (used in addition to a user-specific salt) + scrypt_salt => '<another unique string>', + + # TUWF configuration options, see the TUWF::set() documentation for options. + tuwf => { + db_login => [ 'dbi:Pg:dbname=vndb', 'vndb_site', 'vndb_site' ], + xml_pretty => 0, + log_queries => 0, + debug => 1, + cookie_defaults => { domain => 'localhost', path => '/' }, + mail_sendmail => 'log', + }, + + # Uncomment if you want to test password strength against a dictionary. See + # lib/PWLookup.pm for instructions on how to create the database file. + #password_db => 'data/passwords.dat', + + # Options for Multi, the background server. + Multi => { + # Each module in lib/Multi/ can be enabled and configured here. + Core => { + db_login => { dbname => 'vndb', user => 'vndb_multi', password => 'vndb_multi' }, + }, + #API => {}, + #IRC => { + # nick => 'MyVNDBBot', + # server => 'irc.synirc.net', + # channels => [ '#vndb' ], + # pass => '<nickserv-password>', + # masters => [ 'yorhel!~Ayo@your.hell' ], + #}, + }, +} diff --git a/data/config3_example.pl b/data/config3_example.pl deleted file mode 100644 index d640df5f..00000000 --- a/data/config3_example.pl +++ /dev/null @@ -1,26 +0,0 @@ -{ - # Canonical URL of this site - url => 'http://localhost:3000', - # And of the static files (may be the same as the above url) - url_static => 'http://localhost:3000', - - # TUWF configuration options, see the TUWF::set() documentation for options. - tuwf => { - db_login => [ 'dbi:Pg:dbname=vndb', 'vndb_site', 'vndb_site' ], - xml_pretty => 0, - log_queries => 0, - debug => 1, - cookie_defaults => { domain => 'localhost', path => '/' }, - mail_sendmail => 'log', - }, - - # Configuration of the authentication module (VNDB::Auth) - auth => { - csrf_key => '<some unique string>', - scrypt_salt => '<another unique string>', - }, - - # Uncomment if you want to test password strength against a dictionary. See - # lib/PWLookup.pm for instructions on how to create the database file. - #password_db => 'data/passwords.dat', -} diff --git a/data/config_example.pl b/data/config_example.pl deleted file mode 100644 index 30731d08..00000000 --- a/data/config_example.pl +++ /dev/null @@ -1,41 +0,0 @@ -package VNDB; - -# This file is used to override config options in global.pl. -# You can override anything you want. - -%O = ( - %O, - db_login => [ 'dbi:Pg:dbname=vndb', 'vndb_site', 'vndb_site' ], - #logfile => $ROOT.'/err.log', - xml_pretty => 0, - log_queries => 0, - debug => 1, - cookie_defaults => { domain => 'localhost', path => '/' }, - mail_sendmail => 'log', -); - -%S = ( - %S, - url => 'http://localhost:3000', - url_static => 'http://localhost:3000', - form_salt => '<some unique string>', - scrypt_salt => '<another unique string>', - # Uncomment if you want to test password strength against a dictionary. See - # lib/PWLookup.pm for instructions on how to create the database file. - #password_db => $ROOT.'/data/passwords.dat', -); - -$M{db_login} = { dbname => 'vndb', user => 'vndb_multi', password => 'vndb_multi' }; - -# Uncomment to enable certain features of Multi - -#$M{modules}{API} = {}; -#$M{modules}{APIDump} = {}; - -#$M{modules}{IRC} = { -# nick => 'MyVNDBBot', -# server => 'irc.synirc.net', -# channels => [ '#vndb' ], -# pass => '<nickserv-password>', -# masters => [ 'yorhel!~Ayo@your.hell' ], -#}; diff --git a/data/global.pl b/data/global.pl deleted file mode 100644 index 80051235..00000000 --- a/data/global.pl +++ /dev/null @@ -1,107 +0,0 @@ - -package VNDB; - -use utf8; -use strict; -use warnings; -use Tie::IxHash; - -our $ROOT; - -# Convenient wrapper to create an ordered hash -sub ordhash { my %x; tie %x, 'Tie::IxHash', @_; \%x } - - -# options for TUWF -our %O = ( - db_login => [ 'dbi:Pg:dbname=vndb', 'vndb_site', 'passwd' ], - debug => 1, - logfile => $ROOT.'/data/log/vndb.log', - cookie_prefix => 'vndb_', - cookie_defaults => { - domain => '.vndb.org', - path => '/', - }, -); - - -# VNDB-specific options (object_data) -our %S; -%S = (%S, - version => `cd $ROOT; git describe` =~ /^(.+)$/ && $1, - url => 'http://vndb.org', # Only used by Multi, web pages infer their own address - url_static => 'http://s.vndb.org', - skin_default => 'angel', - placeholder_img => 'http://s.vndb.org/s/angel/bg.jpg', # Used in the og:image meta tag - form_salt => 'a-private-string-here', - scrypt_args => [ 65536, 8, 1 ], # N, r, p - scrypt_salt => 'another-random-string', - regen_static => 0, - source_url => 'http://git.blicky.net/vndb.git/?h=master', - admin_email => 'contact@vndb.org', - login_throttle => [ 24*3600/10, 24*3600 ], # interval between attempts, max burst (10 a day) - scr_size => [ 136, 102 ], # w*h of screenshot thumbnails - ch_size => [ 256, 300 ], # max. w*h of char images - cv_size => [ 256, 400 ], # max. w*h of cover images - # bit flags (Flag 8 was used for staffedit and 256 for affiliates, now free to re-use) - # The 'usermod' flag is hardcoded in sql/func.sql for user_* functions. - permissions => {qw| board 1 boardmod 2 edit 4 tag 16 dbmod 32 tagmod 64 usermod 128 |}, - default_perm => 1+4+16, # Keep synchronised with the default value of users.perm - default_tags_cat=> 'cont,tech', - board_edit_time => 7*24*3600, - atom_feeds => { # num_entries, title, id - announcements => [ 10, 'VNDB Site Announcements', '/t/an' ], - changes => [ 25, 'VNDB Recent Changes', '/hist' ], - posts => [ 25, 'VNDB Recent Posts', '/t' ], - }, - poll_options => 20, # max number of options in discussion board polls - engines => [ grep $_, split /\s*\n\s*/, q{ - BGI/Ethornell - CatSystem2 - codeX RScript - EntisGLS - Ikura GDL - KiriKiri - Majiro - NScripter - QLIE - RPG Maker - RealLive - Ren'Py - Shiina Rio - Unity - YU-RIS - }], - dlsite_url => 'https://www.dlsite.com/%s/work/=/product_id/%%s.html', - denpa_url => 'https://denpasoft.com/products/%s', - jlist_url => 'https://www.jlist.com/%s', - jbox_url => 'https://www.jbox.com/%s', - mg_r18_url => 'https://www.mangagamer.com/r18/detail.php?product_code=%d', - mg_main_url => 'https://www.mangagamer.com/detail.php?product_code=%d', -); - - -# Multi-specific options (Multi also uses some options in %S and %O) -our %M = ( - log_dir => $ROOT.'/data/log', - log_level => 'trace', - modules => { - #API => {}, # disabled by default, not really needed - Feed => {}, - RG => {}, - #Anime => {}, # disabled by default, requires AniDB username/pass - Maintenance => {}, - #IRC => {}, # disabled by default, no need to run an IRC bot when debugging - #Wikidata => {}, # disabled by default, no need to bother the Wikidata API when debugging - #JList => {}, - #MG => {}, - #Denpa => { api => '', user => '', pass => '' }, - }, -); - - -# allow the settings to be overwritten in config.pl -require $ROOT.'/data/config.pl' if -f $ROOT.'/data/config.pl'; - -1; - diff --git a/lib/Multi/API.pm b/lib/Multi/API.pm index a93cd3ff..3f05b6c8 100644 --- a/lib/Multi/API.pm +++ b/lib/Multi/API.pm @@ -17,6 +17,7 @@ use Crypt::URandom 'urandom'; use Crypt::ScryptKDF 'scrypt_raw';; use VNDBUtil 'normalize_query', 'norm_ip'; use VNDB::Types; +use VNDB::Config; use JSON::XS; use PWLookup; @@ -32,7 +33,7 @@ sub FALSE () { JSON::XS::false } my %O = ( port => 19534, tls_port => 19535, # Only used when tls_options is set - logfile => "$VNDB::M{log_dir}/api.log", + logfile => config->{Multi}{Core}{log_dir}.'/api.log', conn_per_ip => 10, max_results => 25, # For get vn/release/producer/character max_results_lists => 100, # For get votelist/vnlist/wishlist @@ -277,7 +278,7 @@ sub login { } else { $arg->{username} = lc $arg->{username}; return cerr $c, auth => "Password too weak, please log in on the site and change your password" - if $VNDB::S{password_db} && PWLookup::lookup($VNDB::S{password_db}, $arg->{password}); + if config->{password_db} && PWLookup::lookup(config->{password_db}, $arg->{password}); } login_auth($c, $arg); @@ -291,7 +292,7 @@ sub login_auth { cpg $c, 'SELECT extract(\'epoch\' from timeout) FROM login_throttle WHERE ip = $1', [ norm_ip($c->{ip}) ], sub { my $tm = $_[0]->nRows ? $_[0]->value(0,0) : AE::time; return cerr $c, auth => "Too many failed login attempts" - if $tm-AE::time() > $VNDB::S{login_throttle}[1]; + if $tm-AE::time() > config->{login_throttle}[1]; # Fetch user info cpg $c, 'SELECT id, encode(user_getscryptargs(id), \'hex\') FROM users WHERE username = $1', [ $arg->{username} ], sub { @@ -311,7 +312,7 @@ sub login_verify { my $token = urandom(20); my($N, $r, $p, $salt) = unpack 'NCCa8', pack 'H*', $sargs; - my $passwd = pack 'NCCa8a*', $N, $r, $p, $salt, scrypt_raw(encode_utf8($arg->{password}), $VNDB::S{scrypt_salt} . $salt, $N, $r, $p, 32); + my $passwd = pack 'NCCa8a*', $N, $r, $p, $salt, scrypt_raw(encode_utf8($arg->{password}), config->{scrypt_salt} . $salt, $N, $r, $p, 32); cpg $c, 'SELECT user_login($1, decode($2, \'hex\'), decode($3, \'hex\'))', [ $uid, unpack('H*', $passwd), unpack('H*', $token) ], sub { if($_[0]->nRows == 1 && ($_[0]->value(0,0)||'') =~ /t/) { @@ -322,7 +323,7 @@ sub login_verify { pg_cmd 'SELECT user_logout($1, decode($2, \'hex\'))', [ $uid, unpack('H*', $token) ]; cres $c, ['ok'], 'Successful login by %s (%s) using client "%s" ver. %s', $arg->{username}, $c->{uid}, $c->{client}, $c->{clientver}; } else { - my @a = ( $tm + $VNDB::S{login_throttle}[0], norm_ip($c->{ip}) ); + my @a = ( $tm + config->{login_throttle}[0], norm_ip($c->{ip}) ); pg_cmd 'UPDATE login_throttle SET timeout = to_timestamp($1) WHERE ip = $2', \@a; pg_cmd 'INSERT INTO login_throttle (ip, timeout) SELECT $2, to_timestamp($1) WHERE NOT EXISTS(SELECT 1 FROM login_throttle WHERE ip = $2)', \@a; cerr $c, auth => "Wrong password for user '$arg->{username}'"; @@ -425,7 +426,7 @@ my %GET_VN = ( encubed => delete($_[0]{l_encubed})||undef, renai => delete($_[0]{l_renai}) ||undef }; - $_[0]{image} = $_[0]{image} ? sprintf '%s/cv/%02d/%d.jpg', $VNDB::S{url_static}, $_[0]{image}%100, $_[0]{image} : undef; + $_[0]{image} = $_[0]{image} ? sprintf '%s/cv/%02d/%d.jpg', config->{url_static}, $_[0]{image}%100, $_[0]{image} : undef; }, }, stats => { @@ -492,7 +493,7 @@ my %GET_VN = ( $i->{screens} = [ grep $i->{id} == $_->{vid}, @$n ]; } for (@$n) { - $_->{image} = sprintf '%s/sf/%02d/%d.jpg', $VNDB::S{url_static}, $_->{image}%100, $_->{image}; + $_->{image} = sprintf '%s/sf/%02d/%d.jpg', config->{url_static}, $_->{image}%100, $_->{image}; $_->{rid} *= 1; $_->{nsfw} = $_->{nsfw} =~ /t/ ? TRUE : FALSE; $_->{width} *= 1; @@ -814,7 +815,7 @@ my %GET_CHARACTER = ( select => 'c.alias AS aliases, c.image, c."desc" AS description', proc => sub { $_[0]{aliases} ||= undef; - $_[0]{image} = $_[0]{image} ? sprintf '%s/ch/%02d/%d.jpg', $VNDB::S{url_static}, $_[0]{image}%100, $_[0]{image} : undef; + $_[0]{image} = $_[0]{image} ? sprintf '%s/ch/%02d/%d.jpg', config->{url_static}, $_[0]{image}%100, $_[0]{image} : undef; $_[0]{description} ||= undef; }, }, diff --git a/lib/Multi/Core.pm b/lib/Multi/Core.pm index 3656f5b7..f8b277bf 100644 --- a/lib/Multi/Core.pm +++ b/lib/Multi/Core.pm @@ -14,6 +14,7 @@ use Pg::PQ ':pgres'; use DBI; use POSIX 'setsid', 'pause', 'SIGUSR1'; use Exporter 'import'; +use VNDB::Config; our @EXPORT = qw|pg pg_cmd pg_expect schedule push_watcher throttle|; @@ -73,7 +74,7 @@ sub daemon_done { sub load_pg { $PG = AnyEvent::Pg::Pool->new( - $VNDB::M{db_login}, + config->{Multi}{Core}{db_login}, timeout => 600, # Some maintenance queries can take a while to run... on_error => sub { die "Lost connection to PostgreSQL\n"; }, on_connect_error => sub { die "Lost connection to PostgreSQL\n"; }, @@ -90,8 +91,9 @@ sub load_pg { sub load_mods { - for(keys %{$VNDB::M{modules}}) { - my($mod, $args) = ($_, $VNDB::M{modules}{$_}); + for(keys %{ config->{Multi} }) { + next if /^Core$/; + my($mod, $args) = ($_, config->{Multi}{$_}); next if !$args || ref($args) ne 'HASH'; require "Multi/$mod.pm"; # I'm surprised the strict pagma isn't complaining about this @@ -104,8 +106,9 @@ sub unload { AE::log info => 'Shutting down'; @watchers = (); - for(keys %{$VNDB::M{modules}}) { - my($mod, $args) = ($_, $VNDB::M{modules}{$_}); + for(keys %{ config->{Multi} }) { + next if /^Core$/; + my($mod, $args) = ($_, config->{Multi}{$_}); next if !$args || ref($args) ne 'HASH'; no strict 'refs'; ${"Multi::$mod\::"}{unload} && "Multi::$mod"->unload(); @@ -115,14 +118,14 @@ sub unload { sub run { my $p = shift; - $pidfile = "$VNDB::ROOT/data/multi.pid"; + $pidfile = config->{root}."/data/multi.pid"; die "PID file already exists\n" if -e $pidfile; $stopcv = AE::cv; - AnyEvent::Log::ctx('Multi')->attach(AnyEvent::Log::Ctx->new(level => $VNDB::M{log_level}, # log_to_file => $VNDB::M{log_dir}.'/multi.log')); + AnyEvent::Log::ctx('Multi')->attach(AnyEvent::Log::Ctx->new(level => config->{Multi}{Core}{log_level}||'trace', # Don't use log_to_file, it doesn't accept perl's unicode strings (and, in fact, crashes on them without logging anything). log_cb => sub { - open(my $F, '>>:utf8', $VNDB::M{log_dir}.'/multi.log'); + open(my $F, '>>:utf8', config->{Multi}{Core}{log_dir}.'/multi.log'); print $F $_[0]; } )); @@ -132,7 +135,7 @@ sub run { load_pg; load_mods; daemon_done; - AE::log info => "Starting Multi $VNDB::S{version}"; + AE::log info => "Starting Multi ".config->{version}; push_watcher(schedule(60, 10*60, \&throttle_gc)); $stopcv->recv; diff --git a/lib/Multi/DLsite.pm b/lib/Multi/DLsite.pm index 8c0871bd..a329d2b7 100644 --- a/lib/Multi/DLsite.pm +++ b/lib/Multi/DLsite.pm @@ -6,6 +6,7 @@ use utf8; use Encode 'decode_utf8'; use Multi::Core; use AnyEvent::HTTP; +use VNDB::Config; my %C = ( @@ -17,7 +18,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{clean_timeout}, sub { diff --git a/lib/Multi/Denpa.pm b/lib/Multi/Denpa.pm index c9139cf2..69fe9292 100644 --- a/lib/Multi/Denpa.pm +++ b/lib/Multi/Denpa.pm @@ -6,6 +6,7 @@ use Multi::Core; use AnyEvent::HTTP; use JSON::XS 'decode_json'; use MIME::Base64 'encode_base64'; +use VNDB::Config; my %C = ( @@ -18,7 +19,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{check_timeout}, sub { diff --git a/lib/Multi/Feed.pm b/lib/Multi/Feed.pm index 76a7e459..626e837b 100644 --- a/lib/Multi/Feed.pm +++ b/lib/Multi/Feed.pm @@ -11,6 +11,7 @@ use TUWF::XML; use Multi::Core; use POSIX 'strftime'; use VNDB::BBCode; +use VNDB::Config; my %stats; # key = feed, value = [ count, total, max ] @@ -39,8 +40,8 @@ sub generate { WHERE NOT t.hidden AND NOT t.private ORDER BY t.id DESC LIMIT $1}, - [$VNDB::S{atom_feeds}{announcements}[0]], - sub { write_atom(announcements => @_) }; + [10], + sub { write_atom(announcements => '/t/an', 'VNDB Site Announcements', @_) }; # changes pg_cmd q{ @@ -59,8 +60,8 @@ sub generate { WHERE c.requester <> 1 ORDER BY c.id DESC LIMIT $1}, - [$VNDB::S{atom_feeds}{changes}[0]], - sub { write_atom(changes => @_); }; + [25], + sub { write_atom(changes => '/hist', 'VNDB Recent Changes', @_); }; # posts pg_cmd q{ @@ -72,13 +73,13 @@ sub generate { WHERE NOT tp.hidden AND NOT t.hidden AND NOT t.private ORDER BY tp.date DESC LIMIT $1}, - [$VNDB::S{atom_feeds}{posts}[0]], - sub { write_atom(posts => @_); }; + [25], + sub { write_atom(posts => '/t', 'VNDB Recent Posts', @_); }; } sub write_atom { - my($feed, $res, $sqltime) = @_; + my($feed, $path, $title, $res, $sqltime) = @_; return if pg_expect $res, 1; my $start = AE::time; @@ -93,33 +94,33 @@ sub write_atom { my $data; my $x = TUWF::XML->new(write => sub { $data .= shift }, pretty => 2); $x->xml(); - $x->tag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => $VNDB::S{url}.'/'); - $x->tag(title => $VNDB::S{atom_feeds}{$feed}[1]); + $x->tag(feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => config->{url}.'/'); + $x->tag(title => $title); $x->tag(updated => datetime($updated)); - $x->tag(id => $VNDB::S{url}.$VNDB::S{atom_feeds}{$feed}[2]); - $x->tag(link => rel => 'self', type => 'application/atom+xml', href => "$VNDB::S{url}/feeds/$feed.atom", undef); - $x->tag(link => rel => 'alternate', type => 'text/html', href => $VNDB::S{url}.$VNDB::S{atom_feeds}{$feed}[2], undef); + $x->tag(id => config->{url}.$path); + $x->tag(link => rel => 'self', type => 'application/atom+xml', href => config->{url}."/feeds/$feed.atom", undef); + $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$path, undef); for(@r) { $x->tag('entry'); - $x->tag(id => $VNDB::S{url}.$_->{id}); + $x->tag(id => config->{url}.$_->{id}); $x->tag(title => $_->{title}); $x->tag(updated => datetime($_->{updated} || $_->{published})); $x->tag(published => datetime($_->{published})) if $_->{published}; if($_->{username}) { $x->tag('author'); $x->tag(name => $_->{username}); - $x->tag(uri => $VNDB::S{url}.'/u'.$_->{uid}) if $_->{uid}; + $x->tag(uri => config->{url}.'/u'.$_->{uid}) if $_->{uid}; $x->end; } - $x->tag(link => rel => 'alternate', type => 'text/html', href => $VNDB::S{url}.$_->{id}, undef); + $x->tag(link => rel => 'alternate', type => 'text/html', href => config->{url}.$_->{id}, undef); $x->tag('summary', type => 'html', bb2html $_->{summary}) if $_->{summary}; $x->end('entry'); } $x->end('feed'); - open my $f, '>:utf8', "$VNDB::ROOT/www/feeds/$feed.atom" || die $!; + open my $f, '>:utf8', config->{root}."/www/feeds/$feed.atom" || die $!; print $f $data; close $f; @@ -139,7 +140,7 @@ sub stats { my $v = $stats{$_}; next if !$v->[0]; AE::log info => sprintf 'Stats summary for %16s.atom: total:%5dms, avg:%4dms, max:%4dms, size: %.1fkB', - $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s "$VNDB::ROOT/www/feeds/$_.atom")/1024; + $_, $v->[1], $v->[1]/$v->[0], $v->[2], (-s config->{root}."/www/feeds/$_.atom")/1024; } %stats = (); } diff --git a/lib/Multi/IRC.pm b/lib/Multi/IRC.pm index b9b8a035..503a1543 100644 --- a/lib/Multi/IRC.pm +++ b/lib/Multi/IRC.pm @@ -11,6 +11,7 @@ use Multi::Core; use AnyEvent::IRC::Client; use AnyEvent::IRC::Util 'prefix_nick'; use VNDBUtil 'normalize_query'; +use VNDB::Config; use TUWF::Misc 'uri_escape'; use POSIX 'strftime'; use Encode 'decode_utf8', 'encode_utf8'; @@ -142,12 +143,7 @@ sub set_cbs { reconnect(); }); - #$irc->reg_cb(read => sub { - # require Data::Dumper; - # AE::log trace => "Received: ".Data::Dumper::Dumper($_[1]); - #}); - - $irc->ctcp_auto_reply(VERSION => ['VERSION', "$O{ircname}:$VNDB::S{version}:AnyEvent"]); + $irc->ctcp_auto_reply(VERSION => ['VERSION', join ':', $O{ircname}, config->{version}, 'AnyEvent']); $irc->ctcp_auto_reply(USERINFO => ['USERINFO', ":$O{ircname}"]); $irc->reg_cb(publicmsg => sub { my @a = (prefix_nick($_[2]->{prefix}), $_[1], $_[2]->{params}[1]); command(@a) || vndbid(@a); }); @@ -162,7 +158,7 @@ sub set_logger { my $l = sub { my($chan, $msg, @arg) = @_; return if !grep $chan eq $_, @{$O{channels}}; - open my $F, '>>', "$VNDB::M{log_dir}/$chan.log" or die $!; + open my $F, '>>', config->{Multi}{Core}{log_dir}."/$chan.log" or die $!; print $F strftime('%Y-%m-%d %H:%M:%S', localtime).' '.sprintf($msg, @arg)."\n"; }; @@ -268,7 +264,7 @@ sub formatid { ) if defined $_->{comments}; # (always) @ URL - push @msg, $c."@ $NORMAL$LIGHT_GREY$VNDB::S{url}/$id$NORMAL"; + push @msg, $c."@ $NORMAL$LIGHT_GREY".config->{url}."/$id$NORMAL"; # now post it $irc->send_msg(PRIVMSG => $dest, encode_utf8 join ' ', @msg); @@ -392,7 +388,7 @@ my %cmds = ( info => [ 0, 0, sub { $irc->send_msg(PRIVMSG => $_[1], - 'Hi! I am HMX-12 Multi '.$VNDB::S{version}.', the IRC bot of '.$VNDB::S{url}.'/, written by the great master Yorhel!'); + 'Hi! I am HMX-12 Multi '.config->{version}.', the IRC bot of '.config->{url}.'/, written by the great master Yorhel!'); }], list => [ 0, 0, sub { @@ -422,7 +418,7 @@ vn => [ 0, 0, sub { return if pg_expect $res, 1; return $irc->send_msg(PRIVMSG => $chan, 'No visual novels found.') if !$res->nRows; return $irc->send_msg(PRIVMSG => $chan, - sprintf 'Too many results found, see %s/v/all?q=%s', $VNDB::S{url}, uri_escape($q)) if $res->nRows > 5; + sprintf 'Too many results found, see %s/v/all?q=%s', config->{url}, uri_escape($q)) if $res->nRows > 5; formatid([$res->rowsAsHashes()], $chan, 0); }; }], @@ -441,7 +437,7 @@ p => [ 0, 0, sub { return if pg_expect $res, 1; return $irc->send_msg(PRIVMSG => $chan, 'No producers novels found.') if !$res->nRows; return $irc->send_msg(PRIVMSG => $chan, - sprintf 'Too many results found, see %s/p/all?q=%s', $VNDB::S{url}, uri_escape($q)) if $res->nRows > 5; + sprintf 'Too many results found, see %s/p/all?q=%s', config->{url}, uri_escape($q)) if $res->nRows > 5; formatid([$res->rowsAsHashes()], $chan, 0); }; }], diff --git a/lib/Multi/JList.pm b/lib/Multi/JList.pm index c3842b8b..c449532a 100644 --- a/lib/Multi/JList.pm +++ b/lib/Multi/JList.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Multi::Core; use AnyEvent::HTTP; +use VNDB::Config; my %C = ( @@ -16,7 +17,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{clean_timeout}, sub { diff --git a/lib/Multi/MG.pm b/lib/Multi/MG.pm index 44a641ed..81659153 100644 --- a/lib/Multi/MG.pm +++ b/lib/Multi/MG.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Multi::Core; use AnyEvent::HTTP; +use VNDB::Config; my %C = ( @@ -16,7 +17,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{clean_timeout}, sub { diff --git a/lib/Multi/Maintenance.pm b/lib/Multi/Maintenance.pm index 3739af1f..4281ff67 100644 --- a/lib/Multi/Maintenance.pm +++ b/lib/Multi/Maintenance.pm @@ -10,6 +10,7 @@ use warnings; use Multi::Core; use PerlIO::gzip; use VNDBUtil 'normalize_titles'; +use VNDB::Config; my $monthly; @@ -134,20 +135,20 @@ my %monthlies = ( sub logrotate { - my $dir = sprintf '%s/old', $VNDB::M{log_dir}; + my $dir = sprintf '%s/old', config->{Multi}{Core}{log_dir}; mkdir $dir if !-d $dir; - for (glob sprintf '%s/*', $VNDB::M{log_dir}) { + for (glob sprintf '%s/*', config->{Multi}{Core}{log_dir}) { next if /^\./ || /~$/ || !-f; my $f = /([^\/]+)$/ ? $1 : $_; my $n = sprintf '%s/%s.%04d-%02d-%02d.gz', $dir, $f, (localtime)[5]+1900, (localtime)[4]+1, (localtime)[3]; return if -f $n; - open my $I, '<', sprintf '%s/%s', $VNDB::M{log_dir}, $f; + open my $I, '<', sprintf '%s/%s', config->{Multi}{Core}{log_dir}, $f; open my $O, '>:gzip', $n; print $O $_ while <$I>; close $O; close $I; - open $I, '>', sprintf '%s/%s', $VNDB::M{log_dir}, $f; + open $I, '>', sprintf '%s/%s', config->{Multi}{Core}{log_dir}, $f; close $I; } AE::log info => 'Logs rotated.'; diff --git a/lib/Multi/PlayAsia.pm b/lib/Multi/PlayAsia.pm index 49128621..8a3249b8 100644 --- a/lib/Multi/PlayAsia.pm +++ b/lib/Multi/PlayAsia.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Multi::Core; use AnyEvent::HTTP; +use VNDB::Config; my %C = ( api => '', @@ -15,7 +16,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Affiliate Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Affiliate Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{sync_gtin_timeout}, \&sync_gtin; diff --git a/lib/Multi/Wikidata.pm b/lib/Multi/Wikidata.pm index b74c3bc2..a4abf5bb 100644 --- a/lib/Multi/Wikidata.pm +++ b/lib/Multi/Wikidata.pm @@ -10,6 +10,7 @@ use warnings; use Multi::Core; use JSON::XS 'decode_json'; use AnyEvent::HTTP; +use VNDB::Config; my %C = ( @@ -22,7 +23,7 @@ my %C = ( sub run { shift; - $C{ua} = "VNDB.org Crawler (Multi v$VNDB::S{version}; contact\@vndb.org)"; + $C{ua} = sprintf 'VNDB.org Crawler (Multi v%s; contact@vndb.org)', config->{version}; %C = (%C, @_); push_watcher schedule 0, $C{check_timeout}, \&fetch; diff --git a/lib/VN3/Auth.pm b/lib/VN3/Auth.pm index a5282666..82b080ed 100644 --- a/lib/VN3/Auth.pm +++ b/lib/VN3/Auth.pm @@ -171,9 +171,9 @@ sub _load_session { sub new { bless { - scrypt_salt => 'random string', - scrypt_args => [ 65536, 8, 1 ], - %{ tuwf->conf->{auth}||{} } + scrypt_salt => tuwf->conf->{scrypt_salt}||die(), + scrypt_args => tuwf->conf->{scrypt_args}||[ 65536, 8, 1 ], + csrf_key => tuwf->conf->{form_salt}||die(), }, shift; } diff --git a/lib/VNDB/Config.pm b/lib/VNDB/Config.pm new file mode 100644 index 00000000..9c9a3a63 --- /dev/null +++ b/lib/VNDB/Config.pm @@ -0,0 +1,86 @@ +package VNDB::Config; + +use strict; +use warnings; +use Exporter 'import'; +our @EXPORT = ('config'); + +my $ROOT = $INC{'VNDB/Config.pm'} =~ s{/lib/VNDB/Config\.pm$}{}r; + +# Default config options +my $config = { + url => 'http://localhost:3000', + + tuwf => { + db_login => [ 'dbi:Pg:dbname=vndb', 'vndb_site', undef ], + cookie_prefix => 'vndb_', + }, + + skin_default => 'angel', + placeholder_img => 'http://s.vndb.org/s/angel/bg.jpg', # Used in the og:image meta tag + scrypt_args => [ 65536, 8, 1 ], # N, r, p + scrypt_salt => 'another-random-string', + form_salt => 'a-private-string-here', + source_url => 'http://git.blicky.net/vndb.git/?h=master', + admin_email => 'contact@vndb.org', + login_throttle => [ 24*3600/10, 24*3600 ], # interval between attempts, max burst (10 a day) + default_tags_cat => 'cont,tech', # Default visible tag categories + board_edit_time => 7*24*3600, # Time after which posts become immutable + poll_options => 20, # max number of options in discussion board polls + + engines => [ grep $_, split /\s*\n\s*/, q{ + BGI/Ethornell + CatSystem2 + codeX RScript + EntisGLS + Ikura GDL + KiriKiri + Majiro + NScripter + QLIE + RPG Maker + RealLive + Ren'Py + Shiina Rio + Unity + YU-RIS + }], + + dlsite_url => 'https://www.dlsite.com/%s/work/=/product_id/%%s.html', + denpa_url => 'https://denpasoft.com/products/%s', + jlist_url => 'https://www.jlist.com/%s', + jbox_url => 'https://www.jbox.com/%s', + mg_r18_url => 'https://www.mangagamer.com/r18/detail.php?product_code=%d', + mg_main_url => 'https://www.mangagamer.com/detail.php?product_code=%d', + + Multi => { + Core => {}, + Feed => {}, + Maintenance => {}, + RG => {}, + }, +}; + + +my $config_file = do $ROOT.'/data/conf.pl'; +my $config_merged; + +sub config { + $config_merged ||= do { + my $c = $config; + $c->{$_} = $config_file->{$_} for grep !/^(Multi|tuwf)$/, keys %$config_file; + $c->{Multi}{$_} = $config_file->{Multi}{$_} for keys %{ $config_file->{Multi} || {} }; + $c->{tuwf}{$_} = $config_file->{tuwf}{$_} for keys %{ $config_file->{tuwf} || {} }; + + $c->{url_static} ||= $c->{url}; + $c->{version} ||= `git -C "$ROOT" describe` =~ /^(.+)$/ && $1; + $c->{root} = $ROOT; + $c->{Multi}{Core}{log_level} ||= 'debug'; + $c->{Multi}{Core}{log_dir} ||= $ROOT.'/data/log'; + $c + }; + $config_merged +} + +1; + diff --git a/lib/VNDB/Handler/Misc.pm b/lib/VNDB/Handler/Misc.pm index d1cba2a4..81a8993d 100644 --- a/lib/VNDB/Handler/Misc.pm +++ b/lib/VNDB/Handler/Misc.pm @@ -42,7 +42,7 @@ sub homepage { 'og:description' => $desc, }; - $self->htmlHeader(title => $title, feeds => [ keys %{$self->{atom_feeds}} ], metadata => $metadata); + $self->htmlHeader(title => $title, feeds => 1, metadata => $metadata); div class => 'mainbox'; h1 $title; diff --git a/lib/VNDB/Util/Auth.pm b/lib/VNDB/Util/Auth.pm index 0093bf2d..c335041a 100644 --- a/lib/VNDB/Util/Auth.pm +++ b/lib/VNDB/Util/Auth.pm @@ -156,7 +156,7 @@ sub authInfo { # returns whether the currently loggedin or anonymous user can perform -# a certain action. Argument is the action name as defined in global.pl +# a certain action. sub authCan { my($self, $act) = @_; return $self->{_auth} ? $self->{_auth}{perm} & $self->{permissions}{$act} : 0; diff --git a/lib/VNDB/Util/LayoutHTML.pm b/lib/VNDB/Util/LayoutHTML.pm index 5cb266f4..c04fa7dd 100644 --- a/lib/VNDB/Util/LayoutHTML.pm +++ b/lib/VNDB/Util/LayoutHTML.pm @@ -14,7 +14,7 @@ our @EXPORT = qw|htmlHeader htmlFooter|; sub htmlHeader { # %options->{ title, noindex, search, feeds, svg, metadata } my($self, %o) = @_; my $skin = $self->reqGet('skin') || $self->authPref('skin') || $self->{skin_default}; - $skin = $self->{skin_default} if !$self->{skins}{$skin} || !-d "$VNDB::ROOT/static/s/$skin"; + $skin = $self->{skin_default} if !$self->{skins}{$skin} || !-d "$self->{root}/static/s/$skin"; # heading lit '<!DOCTYPE HTML>'; @@ -28,8 +28,11 @@ sub htmlHeader { # %options->{ title, noindex, search, feeds, svg, metadata } (my $css = $self->authPref('customcss')) =~ s/\n/ /g; style type => 'text/css', $css; } - Link rel => 'alternate', type => 'application/atom+xml', href => "/feeds/$_.atom", title => $self->{atom_feeds}{$_}[1] - for ($o{feeds} ? @{$o{feeds}} : ()); + if($o{feeds}) { + Link rel => 'alternate', type => 'application/atom+xml', href => "/feeds/announcements.atom", title => 'Site Announcements'; + Link rel => 'alternate', type => 'application/atom+xml', href => "/feeds/changes.atom", title => 'Recent Changes'; + Link rel => 'alternate', type => 'application/atom+xml', href => "/feeds/posts.atom", title => 'Recent Posts'; + } if(exists $o{metadata}) { # Required fields as per http://op.me/#metadata: og:title, og:type, og:image, og:url diff --git a/util/jsgen.pl b/util/jsgen.pl index 146ea680..f5178d78 100755 --- a/util/jsgen.pl +++ b/util/jsgen.pl @@ -1,18 +1,16 @@ #!/usr/bin/perl -package VNDB; - use strict; use warnings; use Encode 'encode_utf8'; use Cwd 'abs_path'; use JSON::XS; -our($ROOT, %S, %O); +my $ROOT; BEGIN { ($ROOT = abs_path $0) =~ s{/util/jsgen\.pl$}{}; } -require $ROOT.'/data/global.pl'; use lib "$ROOT/lib"; +use VNDB::Config; use VNDB::Types; @@ -37,7 +35,7 @@ sub resolutions { sub vars { my %vars = ( rlist_status => [ map [ $_, $RLIST_STATUS{$_} ], keys %RLIST_STATUS ], - cookie_prefix => $O{cookie_prefix}, + cookie_prefix => config->{tuwf}{cookie_prefix}, age_ratings => [ map [ $_, $AGE_RATING{$_}{txt}], keys %AGE_RATING ], languages => [ map [ $_, $LANGUAGE{$_} ], keys %LANGUAGE ], platforms => [ map [ $_, $PLATFORM{$_} ], keys %PLATFORM ], diff --git a/util/multi.pl b/util/multi.pl index 010951a2..1ad92ef4 100755 --- a/util/multi.pl +++ b/util/multi.pl @@ -1,24 +1,13 @@ #!/usr/bin/perl - -# -# Multi - core namespace for initialisation and global variables -# - -package Multi; - use strict; use warnings; use Cwd 'abs_path'; +my $ROOT; +BEGIN { ($ROOT = abs_path $0) =~ s{/util/multi\.pl$}{} } -our $ROOT; -BEGIN { ($ROOT = abs_path $0) =~ s{/util/multi\.pl$}{}; *VNDB::ROOT = \$ROOT } -use lib $VNDB::ROOT.'/lib'; - +use lib $ROOT.'/lib'; use Multi::Core; -require $VNDB::ROOT.'/data/global.pl'; Multi::Core->run(); - - diff --git a/util/vndb-dev-server.pl b/util/vndb-dev-server.pl index bc23997a..d1d051a8 100755 --- a/util/vndb-dev-server.pl +++ b/util/vndb-dev-server.pl @@ -105,9 +105,7 @@ sub checkmod { $check->($_) for (qw{ util/vndb.pl util/vndb3.pl - data/config.pl - data/config3.pl - data/global.pl + data/conf.pl }); my $ismod = $newlastmod > $lastmod; diff --git a/util/vndb.pl b/util/vndb.pl index c0af72e1..e3c721c8 100755 --- a/util/vndb.pl +++ b/util/vndb.pl @@ -1,45 +1,34 @@ #!/usr/bin/perl - -package VNDB; - use strict; use warnings; - - use Cwd 'abs_path'; -our $ROOT; -BEGIN { ($ROOT = abs_path $0) =~ s{/util/vndb\.pl$}{}; } +use TUWF ':html'; $|=1; # Disable buffering on STDOUT, otherwise vndb-dev-server.pl won't pick up our readyness notification. -use lib $ROOT.'/lib'; - +my $ROOT; +BEGIN { ($ROOT = abs_path $0) =~ s{/util/vndb\.pl$}{}; } -use TUWF ':html'; +use lib $ROOT.'/lib'; use SkinFile; - - -our(%O, %S); +use VNDB::Config; # load the skins -# NOTE: $S{skins} can be modified in data/config.pl, allowing deletion of skins or forcing only one skin my $skin = SkinFile->new("$ROOT/static/s"); -$S{skins} = { map +($_ => [ $skin->get($_, 'name'), $skin->get($_, 'userid') ]), $skin->list }; - - -# load settings from global.pl -require $ROOT.'/data/global.pl'; - - -# automatically regenerate the skins and script.js and whatever else should be done -system "make -sC $ROOT" if $S{regen_static}; +tuwf->{skins} = { map +($_ => [ $skin->get($_, 'name'), $skin->get($_, 'userid') ]), $skin->list }; +# Some global variables +tuwf->{scr_size} = [ 136, 102 ]; # w*h of screenshot thumbnails +tuwf->{ch_size} = [ 256, 300 ]; # max. w*h of char images +tuwf->{cv_size} = [ 256, 400 ]; # max. w*h of cover images +tuwf->{permissions} = {qw| board 1 boardmod 2 edit 4 tag 16 dbmod 32 tagmod 64 usermod 128 |}; +tuwf->{default_perm} = 1+4+16; # Keep synchronised with the default value of users.perm +tuwf->{$_} = config->{$_} for keys %{ config() }; -$TUWF::OBJ->{$_} = $S{$_} for (keys %S); TUWF::set( - %O, + %{ config->{tuwf} }, pre_request_handler => \&reqinit, error_404_handler => \&handle404, log_format => \&logformat, diff --git a/util/vndb3.pl b/util/vndb3.pl index ac78553b..693639b0 100755 --- a/util/vndb3.pl +++ b/util/vndb3.pl @@ -10,13 +10,12 @@ BEGIN { ($ROOT = abs_path $0) =~ s{/util/vndb3\.pl$}{}; } use lib $ROOT.'/lib'; use PWLookup; +use VNDB::Config; $|=1; # Disable buffering on STDOUT, otherwise vndb-dev-server.pl won't pick up our readyness notification. -my $conf = require $ROOT.'/data/config3.pl'; - # Make the configuration available as tuwf->conf -sub TUWF::Object::conf { $conf } +sub TUWF::Object::conf { config } # Make our root path available as tuwf->root @@ -48,7 +47,7 @@ sub TUWF::Object::isUnsafePass { } -TUWF::set %{ $conf->{tuwf} || {} }; +TUWF::set %{ config->{tuwf} || {} }; TUWF::set import_modules => 0; |