summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2020-01-24 16:29:51 +0100
committerYorhel <git@yorhel.nl>2020-02-06 11:02:58 +0100
commitedd812bc90f9111b598efc9f4c25e26801767b20 (patch)
tree266886c3e212066e6851e2d189eb53e0e06fea16
parentbc9bf70662fe35bcb4c6a89965d3b4f06bbbe783 (diff)
Cache Markdown-to-HTML in the database for doc pages
Strips about 500ms off the page generation time for d11. The previous in-memory cache kinda worked, too, but it still often happens that the cache is empty. This cache also applies for docs_hist, so browsing through earlier revisions is now faster as well.
-rw-r--r--lib/VNDB/Func.pm31
-rw-r--r--lib/VNWeb/Docs/Edit.pm3
-rw-r--r--lib/VNWeb/Docs/Lib.pm36
-rw-r--r--lib/VNWeb/Docs/Page.pm11
-rw-r--r--lib/VNWeb/Prelude.pm4
-rw-r--r--util/sql/schema.sql6
-rwxr-xr-xutil/update-docs-html-cache.pl16
-rw-r--r--util/updates/2020-02-06-docs-html-cache.sql5
8 files changed, 66 insertions, 46 deletions
diff --git a/lib/VNDB/Func.pm b/lib/VNDB/Func.pm
index 94d1281e..2a169552 100644
--- a/lib/VNDB/Func.pm
+++ b/lib/VNDB/Func.pm
@@ -18,6 +18,7 @@ our @EXPORT = (@VNDBUtil::EXPORT, 'bb2html', 'bb2text', qw|
json_encode json_decode script_json
form_compare
query_encode
+ md2html
|);
@@ -317,5 +318,35 @@ sub query_encode {
} sort keys %$o;
}
+
+sub md2html {
+ require Text::MultiMarkdown;
+ my $html = Text::MultiMarkdown::markdown(shift, {
+ strip_metadata => 1,
+ img_ids => 0,
+ disable_footnotes => 1,
+ disable_bibliography => 1,
+ });
+
+ # Number sections and turn them into links
+ my($sec, $subsec) = (0,0);
+ $html =~ s{<h([1-2])[^>]+>(.*?)</h\1>}{
+ if($1 == 1) {
+ $sec++;
+ $subsec = 0;
+ qq{<h3><a href="#$sec" name="$sec">$sec. $2</a></h3>}
+ } elsif($1 == 2) {
+ $subsec++;
+ qq|<h4><a href="#$sec.$subsec" name="$sec.$subsec">$sec.$subsec. $2</a></h4>\n|
+ }
+ }ge;
+
+ # Text::MultiMarkdown doesn't handle fenced code blocks properly. The
+ # following solution breaks inline code blocks, but I don't use those anyway.
+ $html =~ s/<code>/<pre>/g;
+ $html =~ s#</code>#</pre>#g;
+ $html
+}
+
1;
diff --git a/lib/VNWeb/Docs/Edit.pm b/lib/VNWeb/Docs/Edit.pm
index 82d9506f..dfab77a3 100644
--- a/lib/VNWeb/Docs/Edit.pm
+++ b/lib/VNWeb/Docs/Edit.pm
@@ -39,6 +39,7 @@ elm_api DocEdit => $FORM_OUT, $FORM_IN, sub {
return elm_Unauth if !can_edit d => $doc;
return elm_Unchanged if !form_changed $FORM_CMP, $data, $doc;
+ $data->{html} = md2html $data->{content};
my($id,undef,$rev) = db_edit d => $doc->{id}, $data;
elm_Redirect "/d$id.$rev";
};
@@ -48,7 +49,7 @@ elm_api Markdown => undef, {
content => { required => 0, default => '' }
}, sub {
return elm_Unauth if !auth->permDbmod;
- elm_Content md2html shift->{content};
+ elm_Content enrich_html md2html shift->{content};
};
diff --git a/lib/VNWeb/Docs/Lib.pm b/lib/VNWeb/Docs/Lib.pm
index 38b84421..e6805d45 100644
--- a/lib/VNWeb/Docs/Lib.pm
+++ b/lib/VNWeb/Docs/Lib.pm
@@ -1,9 +1,8 @@
package VNWeb::Docs::Lib;
use VNWeb::Prelude;
-use Text::MultiMarkdown 'markdown';
-our @EXPORT = qw/md2html/;
+our @EXPORT = qw/enrich_html/;
sub _moderators {
@@ -42,36 +41,11 @@ sub _skincontrib {
}
-sub md2html {
- my $content = shift;
+sub enrich_html {
+ my $html = shift;
- $content =~ s{^:MODERATORS:$}{_moderators}me;
- $content =~ s{^:SKINCONTRIB:$}{_skincontrib}me;
-
- my $html = markdown $content, {
- strip_metadata => 1,
- img_ids => 0,
- disable_footnotes => 1,
- disable_bibliography => 1,
- };
-
- # Number sections and turn them into links
- my($sec, $subsec) = (0,0);
- $html =~ s{<h([1-2])[^>]+>(.*?)</h\1>}{
- if($1 == 1) {
- $sec++;
- $subsec = 0;
- qq{<h3><a href="#$sec" name="$sec">$sec. $2</a></h3>}
- } elsif($1 == 2) {
- $subsec++;
- qq|<h4><a href="#$sec.$subsec" name="$sec.$subsec">$sec.$subsec. $2</a></h4>\n|
- }
- }ge;
-
- # Text::MultiMarkdown doesn't handle fenced code blocks properly. The
- # following solution breaks inline code blocks, but I don't use those anyway.
- $html =~ s/<code>/<pre>/g;
- $html =~ s#</code>#</pre>#g;
+ $html =~ s{^:MODERATORS:}{_moderators}me;
+ $html =~ s{^:SKINCONTRIB:}{_skincontrib}me;
$html
}
diff --git a/lib/VNWeb/Docs/Page.pm b/lib/VNWeb/Docs/Page.pm
index 2225890a..4c12f668 100644
--- a/lib/VNWeb/Docs/Page.pm
+++ b/lib/VNWeb/Docs/Page.pm
@@ -37,19 +37,10 @@ sub _rev_ {
}
-# A little in-memory cache of the rendered HTML for the latest revision of each
-# doc page. md2html() performance is "acceptable" for regular page loads but
-# can still feel a little sluggish.
-my %cache; # chid => html
-
-
TUWF::get qr{/$RE{drev}} => sub {
my $d = db_entry d => tuwf->capture('id'), tuwf->capture('rev');
return tuwf->resNotFound if !$d;
- my $html = $cache{$d->{chid}} || md2html $d->{content};
- $cache{$d->{chid}} ||= $html if $d->{chrev} == $d->{maxrev};
-
framework_ title => $d->{title}, index => 1, type => 'd', dbobj => $d, hiddenmsg => 1,
sub {
_rev_ $d if tuwf->capture('rev');
@@ -57,7 +48,7 @@ TUWF::get qr{/$RE{drev}} => sub {
h1_ $d->{title};
div_ class => 'docs', sub {
_index_;
- lit_ $html;
+ lit_ enrich_html($d->{html} || md2html $d->{content});
clearfloat_;
};
};
diff --git a/lib/VNWeb/Prelude.pm b/lib/VNWeb/Prelude.pm
index aa6d9291..0a596bf6 100644
--- a/lib/VNWeb/Prelude.pm
+++ b/lib/VNWeb/Prelude.pm
@@ -14,7 +14,7 @@
# use VNDB::BBCode;
# use VNDB::Types;
# use VNDB::Config;
-# use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr/;
+# use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr md2html/;
# use VNDB::ExtLinks;
# use VNWeb::Auth;
# use VNWeb::HTML;
@@ -58,7 +58,7 @@ sub import {
use VNDB::BBCode;
use VNDB::Types;
use VNDB::Config;
- use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr/;
+ use VNDB::Func qw/fmtdate fmtage fmtvote fmtspoil fmtmedia minage query_encode lang_attr md2html/;
use VNDB::ExtLinks;
use VNWeb::Auth;
use VNWeb::HTML;
diff --git a/util/sql/schema.sql b/util/sql/schema.sql
index f2cc611c..4ac331c0 100644
--- a/util/sql/schema.sql
+++ b/util/sql/schema.sql
@@ -188,14 +188,16 @@ CREATE TABLE docs ( -- dbentry_type=d
locked boolean NOT NULL DEFAULT FALSE,
hidden boolean NOT NULL DEFAULT FALSE,
title varchar(200) NOT NULL DEFAULT '', -- [pub]
- content text NOT NULL DEFAULT '' -- [pub]
+ content text NOT NULL DEFAULT '', -- [pub]
+ html text -- cache, can be manually updated with util/update-docs-html-cache.pl
);
-- docs_hist
CREATE TABLE docs_hist (
chid integer NOT NULL PRIMARY KEY,
title varchar(200) NOT NULL DEFAULT '',
- content text NOT NULL DEFAULT ''
+ content text NOT NULL DEFAULT '',
+ html text -- cache
);
-- login_throttle
diff --git a/util/update-docs-html-cache.pl b/util/update-docs-html-cache.pl
new file mode 100755
index 00000000..8ef0696e
--- /dev/null
+++ b/util/update-docs-html-cache.pl
@@ -0,0 +1,16 @@
+#!/usr/bin/perl
+
+use v5.24;
+use warnings;
+use DBI;
+use Cwd 'abs_path';
+
+my $ROOT;
+BEGIN { ($ROOT = abs_path $0) =~ s{/util/update-docs-html-cache\.pl$}{}; }
+use lib "$ROOT/lib";
+use VNDB::Func 'md2html';
+
+my $db = DBI->connect('dbi:Pg:dbname=vndb', 'vndb', undef, { RaiseError => 1, AutoCommit => 0 });
+$db->do('UPDATE docs_hist SET html = ? WHERE chid = ?', undef, md2html($_->[1]), $_->[0]) for $db->selectall_array('SELECT chid, content FROM docs_hist');
+$db->do('UPDATE docs SET html = ? WHERE id = ?', undef, md2html($_->[1]), $_->[0]) for $db->selectall_array('SELECT id, content FROM docs' );
+$db->commit;
diff --git a/util/updates/2020-02-06-docs-html-cache.sql b/util/updates/2020-02-06-docs-html-cache.sql
new file mode 100644
index 00000000..e7b1540e
--- /dev/null
+++ b/util/updates/2020-02-06-docs-html-cache.sql
@@ -0,0 +1,5 @@
+-- Run 'make' before this script
+-- Run 'util/update-docs-html-cache.pl' after this script
+ALTER TABLE docs ADD COLUMN html text;
+ALTER TABLE docs_hist ADD COLUMN html text;
+\i util/sql/editfunc.sql