diff options
author | Yorhel <git@yorhel.nl> | 2020-01-24 16:29:51 +0100 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2020-02-06 11:02:58 +0100 |
commit | edd812bc90f9111b598efc9f4c25e26801767b20 (patch) | |
tree | 266886c3e212066e6851e2d189eb53e0e06fea16 | |
parent | bc9bf70662fe35bcb4c6a89965d3b4f06bbbe783 (diff) |
Cache Markdown-to-HTML in the database for doc pages
Strips about 500ms off the page generation time for d11. The previous
in-memory cache kinda worked, too, but it still often happens that the
cache is empty.
This cache also applies for docs_hist, so browsing through earlier
revisions is now faster as well.
-rw-r--r-- | lib/VNDB/Func.pm | 31 | ||||
-rw-r--r-- | lib/VNWeb/Docs/Edit.pm | 3 | ||||
-rw-r--r-- | lib/VNWeb/Docs/Lib.pm | 36 | ||||
-rw-r--r-- | lib/VNWeb/Docs/Page.pm | 11 | ||||
-rw-r--r-- | lib/VNWeb/Prelude.pm | 4 | ||||
-rw-r--r-- | util/sql/schema.sql | 6 | ||||
-rwxr-xr-x | util/update-docs-html-cache.pl | 16 | ||||
-rw-r--r-- | util/updates/2020-02-06-docs-html-cache.sql | 5 |
8 files changed, 66 insertions, 46 deletions
diff --git a/lib/VNDB/Func.pm b/lib/VNDB/Func.pm index 94d1281e..2a169552 100644 --- a/lib/VNDB/Func.pm +++ b/lib/VNDB/Func.pm @@ -18,6 +18,7 @@ our @EXPORT = (@VNDBUtil::EXPORT, 'bb2html', 'bb2text', qw| json_encode json_decode script_json form_compare query_encode + md2html |); @@ -317,5 +318,35 @@ sub query_encode { } sort keys %$o; } + +sub md2html { + require Text::MultiMarkdown; + my $html = Text::MultiMarkdown::markdown(shift, { + strip_metadata => 1, + img_ids => 0, + disable_footnotes => 1, + disable_bibliography => 1, + }); + + # Number sections and turn them into links + my($sec, $subsec) = (0,0); + $html =~ s{<h([1-2])[^>]+>(.*?)</h\1>}{ + if($1 == 1) { + $sec++; + $subsec = 0; + qq{<h3><a href="#$sec" name="$sec">$sec. $2</a></h3>} + } elsif($1 == 2) { + $subsec++; + qq|<h4><a href="#$sec.$subsec" name="$sec.$subsec">$sec.$subsec. $2</a></h4>\n| + } + }ge; + + # Text::MultiMarkdown doesn't handle fenced code blocks properly. The + # following solution breaks inline code blocks, but I don't use those anyway. + $html =~ s/<code>/<pre>/g; + $html =~ s#</code>#</pre>#g; + $html +} + 1; diff --git a/lib/VNWeb/Docs/Edit.pm b/lib/VNWeb/Docs/Edit.pm index 82d9506f..dfab77a3 100644 --- a/lib/VNWeb/Docs/Edit.pm +++ b/lib/VNWeb/Docs/Edit.pm @@ -39,6 +39,7 @@ elm_api DocEdit => $FORM_OUT, $FORM_IN, sub { return elm_Unauth if !can_edit d => $doc; return elm_Unchanged if !form_changed $FORM_CMP, $data, $doc; + $data->{html} = md2html $data->{content}; my($id,undef,$rev) = db_edit d => $doc->{id}, $data; elm_Redirect "/d$id.$rev"; }; @@ -48,7 +49,7 @@ elm_api Markdown => undef, { content => { required => 0, default => '' } }, sub { return elm_Unauth if !auth->permDbmod; - elm_Content md2html shift->{content}; + elm_Content enrich_html md2html shift->{content}; }; diff --git a/lib/VNWeb/Docs/Lib.pm b/lib/VNWeb/Docs/Lib.pm index 38b84421..e6805d45 100644 --- a/lib/VNWeb/Docs/Lib.pm +++ b/lib/VNWeb/Docs/Lib.pm @@ -1,9 +1,8 @@ package VNWeb::Docs::Lib; use VNWeb::Prelude; -use Text::MultiMarkdown 'markdown'; -our @EXPORT = qw/md2html/; +our @EXPORT = qw/enrich_html/; sub _moderators { @@ -42,36 +41,11 @@ sub _skincontrib { } -sub md2html { - my $content = shift; +sub enrich_html { + my $html = shift; - $content =~ s{^:MODERATORS:$}{_moderators}me; - $content =~ s{^:SKINCONTRIB:$}{_skincontrib}me; - - my $html = markdown $content, { - strip_metadata => 1, - img_ids => 0, - disable_footnotes => 1, - disable_bibliography => 1, - }; - - # Number sections and turn them into links - my($sec, $subsec) = (0,0); - $html =~ s{<h([1-2])[^>]+>(.*?)</h\1>}{ - if($1 == 1) { - $sec++; - $subsec = 0; - qq{<h3><a href="#$sec" name="$sec">$sec. $2</a></h3>} - } elsif($1 == 2) { - $subsec++; - qq|<h4><a href="#$sec.$subsec" name="$sec.$subsec">$sec.$subsec. $2</a></h4>\n| - } - }ge; - - # Text::MultiMarkdown doesn't handle fenced code blocks properly. The - # following solution breaks inline code blocks, but I don't use those anyway. - $html =~ s/<code>/<pre>/g; - $html =~ s#</code>#</pre>#g; + $html =~ s{^:MODERATORS:}{_moderators}me; + $html =~ s{^:SKINCONTRIB:}{_skincontrib}me; $html } diff --git a/lib/VNWeb/Docs/Page.pm b/lib/VNWeb/Docs/Page.pm index 2225890a..4c12f668 100644 --- a/lib/VNWeb/Docs/Page.pm +++ b/lib/VNWeb/Docs/Page.pm @@ -37,19 +37,10 @@ sub _rev_ { } -# A little in-memory cache of the rendered HTML for the latest revision of each -# doc page. md2html() performance is "acceptable" for regular page loads but -# can still feel a little sluggish. -my %cache; # chid => html - - TUWF::get qr{/$RE{drev}} => sub { my $d = db_entry d => tuwf->capture('id'), tuwf->capture('rev'); return tuwf->resNotFound if !$d; - my $html = $cache{$d->{chid}} || md2html $d->{content}; - $cache{$d->{chid}} ||= $html if $d->{chrev} == $d->{maxrev}; - framework_ title => $d->{title}, index => 1, type => 'd', dbobj => $d, hiddenmsg => 1, sub { _rev_ $d if tuwf->capture('rev'); @@ -57,7 +48,7 @@ TUWF::get qr{/$RE{drev}} => sub { h1_ $d->{title}; div_ class => 'docs', sub { _index_; - lit_ $html; + lit_ enrich_html($d->{html} || md2html $d->{content}); clearfloat_; }; }; diff --git a/lib/VNWeb/Prelude.pm b/lib/VNWeb/Prelude.pm index aa6d9291..0a596bf6 100644 --- a/lib/VNWeb/Prelude.pm +++ b/lib/VNWeb/Prelude.pm @@ -14,7 +14,7 @@ # use VNDB::BBCode; # use VNDB::Types; # use VNDB::Config; -# use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr/; +# use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr md2html/; # use VNDB::ExtLinks; # use VNWeb::Auth; # use VNWeb::HTML; @@ -58,7 +58,7 @@ sub import { use VNDB::BBCode; use VNDB::Types; use VNDB::Config; - use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr/; + use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr md2html/; use VNDB::ExtLinks; use VNWeb::Auth; use VNWeb::HTML; diff --git a/util/sql/schema.sql b/util/sql/schema.sql index f2cc611c..4ac331c0 100644 --- a/util/sql/schema.sql +++ b/util/sql/schema.sql @@ -188,14 +188,16 @@ CREATE TABLE docs ( -- dbentry_type=d locked boolean NOT NULL DEFAULT FALSE, hidden boolean NOT NULL DEFAULT FALSE, title varchar(200) NOT NULL DEFAULT '', -- [pub] - content text NOT NULL DEFAULT '' -- [pub] + content text NOT NULL DEFAULT '', -- [pub] + html text -- cache, can be manually updated with util/update-docs-html-cache.pl ); -- docs_hist CREATE TABLE docs_hist ( chid integer NOT NULL PRIMARY KEY, title varchar(200) NOT NULL DEFAULT '', - content text NOT NULL DEFAULT '' + content text NOT NULL DEFAULT '', + html text -- cache ); -- login_throttle diff --git a/util/update-docs-html-cache.pl b/util/update-docs-html-cache.pl new file mode 100755 index 00000000..8ef0696e --- /dev/null +++ b/util/update-docs-html-cache.pl @@ -0,0 +1,16 @@ +#!/usr/bin/perl + +use v5.24; +use warnings; +use DBI; +use Cwd 'abs_path'; + +my $ROOT; +BEGIN { ($ROOT = abs_path $0) =~ s{/util/update-docs-html-cache\.pl$}{}; } +use lib "$ROOT/lib"; +use VNDB::Func 'md2html'; + +my $db = DBI->connect('dbi:Pg:dbname=vndb', 'vndb', undef, { RaiseError => 1, AutoCommit => 0 }); +$db->do('UPDATE docs_hist SET html = ? WHERE chid = ?', undef, md2html($_->[1]), $_->[0]) for $db->selectall_array('SELECT chid, content FROM docs_hist'); +$db->do('UPDATE docs SET html = ? WHERE id = ?', undef, md2html($_->[1]), $_->[0]) for $db->selectall_array('SELECT id, content FROM docs' ); +$db->commit; diff --git a/util/updates/2020-02-06-docs-html-cache.sql b/util/updates/2020-02-06-docs-html-cache.sql new file mode 100644 index 00000000..e7b1540e --- /dev/null +++ b/util/updates/2020-02-06-docs-html-cache.sql @@ -0,0 +1,5 @@ +-- Run 'make' before this script +-- Run 'util/update-docs-html-cache.pl' after this script +ALTER TABLE docs ADD COLUMN html text; +ALTER TABLE docs_hist ADD COLUMN html text; +\i util/sql/editfunc.sql |