diff options
author | Yorhel <git@yorhel.nl> | 2013-05-13 15:29:15 +0200 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2013-05-13 16:03:47 +0200 |
commit | bd0e336eebe2136f4f17ef8215238020fa485cea (patch) | |
tree | 63e022e115efae703bfe651d1b9352b0573e53fa | |
parent | 472e7d523a3c21cd1c2850f87c2b70ecd6f393f3 (diff) |
share/db: Use sqlasync_custom() to load the file list in the DB thread
Improves performance a bit on single-core systems. Not as significant as
I had hoped.
-rw-r--r-- | src/share/db.c | 92 |
1 files changed, 54 insertions, 38 deletions
diff --git a/src/share/db.c b/src/share/db.c index fa2fe8d..042e1e1 100644 --- a/src/share/db.c +++ b/src/share/db.c @@ -59,7 +59,7 @@ static void pathids_destroy() { } -static inline uint32_t blob2pathid(uint8_t *blob) { +static inline uint32_t blob2pathid(const uint8_t *blob) { return (((uint32_t)blob[0]) << 24) + (((uint32_t)blob[1]) << 16) + @@ -70,15 +70,15 @@ static inline uint32_t blob2pathid(uint8_t *blob) { -static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, time_t lastmod, const char *tth, kstring_t *buf) { +static void share_db_load_filepath(share_path_t *p, char *path, const char *name, uint64_t size, time_t lastmod, const char *tth, kstring_t *buf) { /* Removing the FilesystemPath from the file path will give the relative * path of the file within the configured share. Prefixing that with the * VirtualPath will give the full virtual path of the file within the * configured share. For example: * - * path = /share/Some/File.mp3 - * p.FilesystemPath = /share -> relative = Some/File.mp3 - * p.VirtualPath = /MySharedFiles -> virtual = /MySharedFiles/Some/File.mp3 + * path = /share/Some + * p.FilesystemPath = /share -> relative = Some + * p.VirtualPath = /MySharedFiles -> virtual = /MySharedFiles/Some * * This only works when symlinks aren't resolved. Which we don't do. */ @@ -92,9 +92,7 @@ static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, t } if(fslen > 1) path = path + fslen; - char *name = strrchr(path, '/'); - *(name++) = 0; - /* Following the example above, path is now '/Some' and name is 'File.mp3' */ + /* path is now relative to the configured share path, but does have a '/' prefix */ share_fl_t *fl = share_fl_create(size, name, buf); fl->lastmod = lastmod; @@ -119,55 +117,73 @@ static void share_db_load_filepath(share_path_t *p, char *path, uint64_t size, t ywarn("Duplicate filenames in %s%s: %s vs. %s", p->props.FilesystemPath, path, buf[0].s, buf[1].s); free(fl); } - - *(name-1) = '/'; } -static void share_db_load_file(sqlasync_result_t *r, kstring_t *buf) { - if(r->col[0].type != SQLITE_TEXT - || r->col[1].type != SQLITE_BLOB || (r->col[1].length % 4) != 0 - || r->col[2].type != SQLITE_INTEGER || r->col[2].val.i64 < 0 - || r->col[3].type != SQLITE_INTEGER || r->col[3].val.i64 < 0 - || r->col[4].type != SQLITE_BLOB || r->col[4].length != 24) +static void share_db_load_file(sqlite3_stmt *st, kstring_t *buf) { + const char *fspath = (const char *)sqlite3_column_text(st, 0); + const uint8_t *paths = sqlite3_column_blob(st, 1); + size_t pathslen = sqlite3_column_bytes(st, 1); + int64_t size = sqlite3_column_type(st, 2) == SQLITE_INTEGER ? sqlite3_column_int64(st, 2) : -1; + int64_t lastmod = sqlite3_column_type(st, 3) == SQLITE_INTEGER ? sqlite3_column_int64(st, 3) : -1; + const char *tth = sqlite3_column_blob(st, 4); + + if(!fspath || *fspath != '/' || !paths || (pathslen % 4) != 0 || size < 0 || + lastmod < 0 || !tth || sqlite3_column_bytes(st, 4) != 24) return; - uint8_t *paths = r->col[1].val.ptr; + char *name = strrchr(fspath, '/'); + buf[2].l = 0; + kputsn(fspath, fspath == name ? 1 : name-fspath, buf+2); + name++; + size_t i; - for(i=0; i<r->col[1].length; i+=4) { + for(i=0; i<pathslen; i+=4) { khiter_t k = kh_get(pathids, pathids, blob2pathid(paths+i)); if(k == kh_end(pathids)) continue; - share_db_load_filepath(kh_value(pathids, k), - r->col[0].val.ptr, r->col[2].val.i64, r->col[3].val.i64, r->col[4].val.ptr, buf); + share_db_load_filepath(kh_value(pathids, k), buf[2].s, name, size, lastmod, tth, buf); } } +/* The file list is loaded using sqlasync_custom(), because passing back + * thousands or millions of query results over an asynchronous queue adds a + * noticable overhead. The share_fl_t structures are now generated in the + * database thread. There is no need for locks in this case; The main thread + * is waiting for this function to complete before it will continue with + * further initialization. */ +static void share_db_load_sql(sqlasync_t *sql, sqlite3 *db, sqlasync_queue_t *q, int val_num, sqlasync_value_t *val) { + sqlite3_stmt *st; + kstring_t buf[3] = {}; /* Three string buffers for use with share_db_load_file() */ + + assert(sqlite3_prepare_v2(db, "SELECT path, pathids, size, lastmodified, tth_root FROM sharefiles", -1, &st, NULL) == SQLITE_OK); + + int r; + while((r = sqlite3_step(st)) == SQLITE_ROW) + share_db_load_file(st, buf); + + sqlite3_reset(st); + sqlite3_finalize(st); + free(buf[0].s); + free(buf[1].s); + free(buf[2].s); + + sqlasync_result_t *res = sqlasync_result_create(r, 1, r != SQLITE_DONE); + if(r != SQLITE_DONE) + res->col[0] = sqlasync_text(SQLASYNC_COPY, sqlite3_errmsg(db)); + sqlasync_queue_result(q, res); +} + + void share_db_load() { pathids_create(); ydebug("Loading file list"); - /* XXX: Some performance is lost in the queueing code of sqlasync. It'd - * help if sqlasync first queued up a bunch of results (without locking) - * and then batch-moved those results to the final queue, in a single - * critical section and with a single pthread_cond_signal() call. I'm not - * sure if that's worth the complexity, however. */ - sqlasync_queue_t *q = sqlasync_queue_buffersize(sqlasync_queue_sync(), 50); - sqlasync_sql(db_sql, q, SQLASYNC_STATIC, - "SELECT path, pathids, size, lastmodified, tth_root FROM sharefiles", 0); - double st = ev_time(); - kstring_t buf[2] = {}; - sqlasync_result_t *r; - while((r = db_get(q, "loading sharefiles")) && r->result == SQLITE_ROW) { - share_db_load_file(r, buf); - sqlasync_result_free(r); - } - sqlasync_result_free(r); + sqlasync_queue_t *q = sqlasync_custom(db_sql, sqlasync_queue_sync(), share_db_load_sql, 0); + sqlasync_result_free(db_get(q, "loading sharefiles")); sqlasync_queue_destroy(q); - free(buf[0].s); - free(buf[1].s); yinfo("File list loaded in %.3f seconds", ev_time()-st); pathids_destroy(); |