share/db: Use sqlasync_custom() to load the file list in the DB thread

Improves performance a bit on single-core systems. Not as significant as
I had hoped.

1 files changed, 54 insertions, 38 deletions

diff --git a/src/share/db.c b/src/share/db.c
index fa2fe8d..042e1e1 100644
--- a/src/share/db.c
+++ b/src/share/db.c
@@ -59,7 +59,7 @@ static void pathids_destroy() {
 }
 
 
-static inline uint32_t blob2pathid(uint8_t *blob) {
+static inline uint32_t blob2pathid(const uint8_t *blob) {
 	return
 		(((uint32_t)blob[0]) << 24) +
 		(((uint32_t)blob[1]) << 16) +
@@ -70,15 +70,15 @@ static inline uint32_t blob2pathid(uint8_t *blob) {
 
 
 
-static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, time_t lastmod, const char *tth, kstring_t *buf) {
+static void share_db_load_filepath(share_path_t *p, char *path, const char *name, uint64_t size, time_t lastmod, const char *tth, kstring_t *buf) {
 	/* Removing the FilesystemPath from the file path will give the relative
 	 * path of the file within the configured share. Prefixing that with the
 	 * VirtualPath will give the full virtual path of the file within the
 	 * configured share. For example:
 	 *
-	 *   path = /share/Some/File.mp3
-	 *   p.FilesystemPath = /share       -> relative = Some/File.mp3
-	 *   p.VirtualPath = /MySharedFiles  -> virtual  = /MySharedFiles/Some/File.mp3
+	 *   path = /share/Some
+	 *   p.FilesystemPath = /share       -> relative = Some
+	 *   p.VirtualPath = /MySharedFiles  -> virtual  = /MySharedFiles/Some
 	 *
 	 * This only works when symlinks aren't resolved. Which we don't do.
 	 */
@@ -92,9 +92,7 @@ static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, t
 	}
 	if(fslen > 1)
 		path = path + fslen;
-	char *name = strrchr(path, '/');
-	*(name++) = 0;
-	/* Following the example above, path is now '/Some' and name is 'File.mp3' */
+	/* path is now relative to the configured share path, but does have a '/' prefix */
 
 	share_fl_t *fl = share_fl_create(size, name, buf);
 	fl->lastmod = lastmod;
@@ -119,55 +117,73 @@ static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, t
 		ywarn("Duplicate filenames in %s%s: %s vs. %s", p->props.FilesystemPath, path, buf[0].s, buf[1].s);
 		free(fl);
 	}
-
-	*(name-1) = '/';
 }
 
 
-static void share_db_load_file(sqlasync_result_t *r, kstring_t *buf) {
-	if(r->col[0].type != SQLITE_TEXT
-			|| r->col[1].type != SQLITE_BLOB || (r->col[1].length % 4) != 0
-			|| r->col[2].type != SQLITE_INTEGER || r->col[2].val.i64 < 0
-			|| r->col[3].type != SQLITE_INTEGER || r->col[3].val.i64 < 0
-			|| r->col[4].type != SQLITE_BLOB || r->col[4].length != 24)
+static void share_db_load_file(sqlite3_stmt *st, kstring_t *buf) {
+	const char *fspath = (const char *)sqlite3_column_text(st, 0);
+	const uint8_t *paths = sqlite3_column_blob(st, 1);
+	size_t pathslen = sqlite3_column_bytes(st, 1);
+	int64_t size = sqlite3_column_type(st, 2) == SQLITE_INTEGER ? sqlite3_column_int64(st, 2) : -1;
+	int64_t lastmod = sqlite3_column_type(st, 3) == SQLITE_INTEGER ? sqlite3_column_int64(st, 3) : -1;
+	const char *tth = sqlite3_column_blob(st, 4);
+
+	if(!fspath || *fspath != '/' || !paths || (pathslen % 4) != 0 || size < 0 ||
+			lastmod < 0 || !tth || sqlite3_column_bytes(st, 4) != 24)
 		return;
 
-	uint8_t *paths = r->col[1].val.ptr;
+	char *name = strrchr(fspath, '/');
+	buf[2].l = 0;
+	kputsn(fspath, fspath == name ? 1 : name-fspath, buf+2);
+	name++;
+
 	size_t i;
-	for(i=0; i<r->col[1].length; i+=4) {
+	for(i=0; i<pathslen; i+=4) {
 		khiter_t k = kh_get(pathids, pathids, blob2pathid(paths+i));
 		if(k == kh_end(pathids))
 			continue;
-		share_db_load_filepath(kh_value(pathids, k),
-				r->col[0].val.ptr, r->col[2].val.i64, r->col[3].val.i64, r->col[4].val.ptr, buf);
+		share_db_load_filepath(kh_value(pathids, k), buf[2].s, name, size, lastmod, tth, buf);
 	}
 }
 
 
+/* The file list is loaded using sqlasync_custom(), because passing back
+ * thousands or millions of query results over an asynchronous queue adds a
+ * noticable overhead.  The share_fl_t structures are now generated in the
+ * database thread.  There is no need for locks in this case; The main thread
+ * is waiting for this function to complete before it will continue with
+ * further initialization. */
+static void share_db_load_sql(sqlasync_t *sql, sqlite3 *db, sqlasync_queue_t *q, int val_num, sqlasync_value_t *val) {
+	sqlite3_stmt *st;
+	kstring_t buf[3] = {}; /* Three string buffers for use with share_db_load_file() */
+
+	assert(sqlite3_prepare_v2(db, "SELECT path, pathids, size, lastmodified, tth_root FROM sharefiles", -1, &st, NULL) == SQLITE_OK);
+
+	int r;
+	while((r = sqlite3_step(st)) == SQLITE_ROW)
+		share_db_load_file(st, buf);
+
+	sqlite3_reset(st);
+	sqlite3_finalize(st);
+	free(buf[0].s);
+	free(buf[1].s);
+	free(buf[2].s);
+
+	sqlasync_result_t *res = sqlasync_result_create(r, 1, r != SQLITE_DONE);
+	if(r != SQLITE_DONE)
+		res->col[0] = sqlasync_text(SQLASYNC_COPY, sqlite3_errmsg(db));
+	sqlasync_queue_result(q, res);
+}
+
+
 void share_db_load() {
 	pathids_create();
 	ydebug("Loading file list");
 
-	/* XXX: Some performance is lost in the queueing code of sqlasync. It'd
-	 * help if sqlasync first queued up a bunch of results (without locking)
-	 * and then batch-moved those results to the final queue, in a single
-	 * critical section and with a single pthread_cond_signal() call. I'm not
-	 * sure if that's worth the complexity, however. */
-	sqlasync_queue_t *q = sqlasync_queue_buffersize(sqlasync_queue_sync(), 50);
-	sqlasync_sql(db_sql, q, SQLASYNC_STATIC,
-			"SELECT path, pathids, size, lastmodified, tth_root FROM sharefiles", 0);
-
 	double st = ev_time();
-	kstring_t buf[2] = {};
-	sqlasync_result_t *r;
-	while((r = db_get(q, "loading sharefiles")) && r->result == SQLITE_ROW) {
-		share_db_load_file(r, buf);
-		sqlasync_result_free(r);
-	}
-	sqlasync_result_free(r);
+	sqlasync_queue_t *q = sqlasync_custom(db_sql, sqlasync_queue_sync(), share_db_load_sql, 0);
+	sqlasync_result_free(db_get(q, "loading sharefiles"));
 	sqlasync_queue_destroy(q);
-	free(buf[0].s);
-	free(buf[1].s);
 
 	yinfo("File list loaded in %.3f seconds", ev_time()-st);
 	pathids_destroy();