diff options
| -rw-r--r-- | GNUmakefile.am | 1 | ||||
| -rw-r--r-- | src/fl_load.c | 377 | ||||
| -rw-r--r-- | src/fl_util.c | 308 | ||||
| -rw-r--r-- | src/ncdc.h | 1 |
4 files changed, 380 insertions, 307 deletions
diff --git a/GNUmakefile.am b/GNUmakefile.am index 70d6f68..83d9abc 100644 --- a/GNUmakefile.am +++ b/GNUmakefile.am @@ -25,6 +25,7 @@ ncdc_SOURCES=\ src/commands.c\ src/db.c\ src/dl.c\ + src/fl_load.c\ src/fl_local.c\ src/fl_save.c\ src/fl_util.c\ diff --git a/src/fl_load.c b/src/fl_load.c new file mode 100644 index 0000000..8729473 --- /dev/null +++ b/src/fl_load.c @@ -0,0 +1,377 @@ +/* ncdc - NCurses Direct Connect client + + Copyright (c) 2011-2012 Yoran Heling + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + + +#include "ncdc.h" +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <bzlib.h> + + +#define S_START 0 // waiting for <FileListing> +#define S_FLOPEN 1 // In a <FileListing ..> +#define S_DIROPEN 2 // In a <Directory ..> +#define S_INDIR 3 // In a <Directory>..</Directory> or <FileListing>..</FileListing> +#define S_FILEOPEN 4 // In a <File ..> +#define S_INFILE 5 // In a <File>..</File> +#define S_UNKNOWN 6 // In some tag we didn't recognize +#define S_END 7 // Received </FileListing> + +struct ctx { + BZFILE *fh_bz; + FILE *fh_f; + gboolean eof; + + gboolean local; + int state; + char *name; + char filetth[24]; + gboolean filehastth; + guint64 filesize; + gboolean dirincomplete; + struct fl_list *root; + struct fl_list *cur; + int unknown_level; +}; + + +static int readcb(void *context, char *buf, int len, GError **err) { + struct ctx *x = context; + + if(x->fh_bz) { + if(x->eof) + return 0; + int bzerr; + int r = BZ2_bzRead(&bzerr, x->fh_bz, buf, len); + if(bzerr != BZ_OK && bzerr != BZ_STREAM_END) { + g_set_error(err, 1, 0, "bzip2 decompression error (%d): %s", bzerr, g_strerror(errno)); + return -1; + } + if(bzerr == BZ_STREAM_END) + x->eof = TRUE; + return r; + + } + + int r = fread(buf, 1, len, x->fh_f); + if(r < 0 && feof(x->fh_f)) + r = 0; + if(r < 0) + g_set_error(err, 1, 0, "Read error: %s", g_strerror(errno)); + return r; +} + + +static int entitycb(void *context, int type, const char *arg1, const char *arg2, GError **err) { + struct ctx *x = context; + //printf("%d,%d: %s, %s\n", x->state, type, arg1, arg2); + switch(x->state) { + + // The first token must always be a <FileListing> + case S_START: + if(type == XMLT_OPEN && strcmp(arg1, "FileListing") == 0) { + x->state = S_FLOPEN; + return 0; + } + break; + + // Any attributes in a <FileListing> are currently ignored. + case S_FLOPEN: + if(type == XMLT_ATTR) + return 0; + if(type == XMLT_ATTDONE) { + x->state = S_INDIR; + return 0; + } + break; + + // Handling the attributes of a Directory element. + case S_DIROPEN: + if(type == XMLT_ATTR && strcmp(arg1, "Name") == 0 && !x->name) { + if(!g_utf8_validate(arg2, -1, NULL)) { + g_set_error(err, 1, 0, "Invalid UTF-8"); + return -1; + } + x->name = g_strdup(arg2); + return 0; + } + if(type == XMLT_ATTDONE) { + if(!x->name) { + g_set_error(err, 1, 0, "Missing Name attribute in Directory element"); + return -1; + } + // Create the directory entry + struct fl_list *new = fl_list_create(x->name, FALSE); + new->isfile = FALSE; + new->sub = g_ptr_array_new_with_free_func(fl_list_free); + fl_list_add(x->cur, new, -1); + x->cur = new; + + g_free(x->name); + x->name = NULL; + x->state = S_INDIR; + return 0; + } + // Ignore unknown or duplicate attributes. + if(type == XMLT_ATTR) + return 0; + break; + + // In a directory listing. + case S_INDIR: + if(type == XMLT_OPEN && strcmp(arg1, "Directory") == 0) { + x->state = S_DIROPEN; + return 0; + } + if(type == XMLT_OPEN && strcmp(arg1, "File") == 0) { + x->state = S_FILEOPEN; + return 0; + } + if(type == XMLT_OPEN) { + x->state = S_UNKNOWN; + x->unknown_level = 1; + return 0; + } + if(type == XMLT_CLOSE) { + char *expect = x->root == x->cur ? "FileListing" : "Directory"; + if(arg1 && strcmp(arg1, expect) != 0) { + g_set_error(err, 1, 0, "Invalid close tag, expected </%s> but got </%s>", expect, arg1); + return -1; + } + fl_list_sort(x->cur); + if(x->cur == x->root) + x->state = S_END; + else + x->cur = x->cur->parent; + return 0; + } + break; + + // Handling the attributes of a File element. (If there are multiple + // attributes with the same name, only the first is used.) + case S_FILEOPEN: + if(type == XMLT_ATTR && strcmp(arg1, "Name") == 0 && !x->name) { + if(!g_utf8_validate(arg2, -1, NULL)) { + g_set_error(err, 1, 0, "Invalid UTF-8"); + return -1; + } + x->name = g_strdup(arg2); + return 0; + } + if(type == XMLT_ATTR && strcmp(arg1, "TTH") == 0 && !x->filehastth) { + if(!istth(arg2)) { + g_set_error(err, 1, 0, "Invalid TTH"); + return -1; + } + base32_decode(arg2, x->filetth); + x->filehastth = TRUE; + return 0; + } + if(type == XMLT_ATTR && strcmp(arg1, "Size") == 0 && x->filesize == G_MAXUINT64) { + char *end = NULL; + x->filesize = g_ascii_strtoull(arg2, &end, 10); + if(!end || *end) { + g_set_error(err, 1, 0, "Invalid file size"); + return -1; + } + return 0; + } + if(type == XMLT_ATTDONE) { + if(!x->name || !x->filehastth || x->filesize == G_MAXUINT64) { + g_set_error(err, 1, 0, "Missing %s attribute in File element", + !x->name ? "Name" : !x->filehastth ? "TTH" : "Size"); + return -1; + } + // Create the file entry + struct fl_list *new = fl_list_create(x->name, x->local); + new->isfile = TRUE; + new->size = x->filesize; + new->hastth = TRUE; + memcpy(new->tth, x->filetth, 24); + fl_list_add(x->cur, new, -1); + + x->filehastth = FALSE; + x->filesize = G_MAXUINT64; + g_free(x->name); + x->name = NULL; + x->state = S_INFILE; + return 0; + } + // Ignore unknown or duplicate attributes. + if(type == XMLT_ATTR) + return 0; + break; + + // In a File element. Nothing is allowed here exept a close of the File + // element. (Really?) + case S_INFILE: + if(type == XMLT_CLOSE && (!arg1 || strcmp(arg1, "File") == 0)) { + x->state = S_INDIR; + return 0; + } + break; + + // No idea in what kind of tag we are, just count start/end tags so we can + // continue parsing when we're out of this unknown tag. + case S_UNKNOWN: + if(type == XMLT_OPEN) + x->unknown_level++; + else if(type == XMLT_CLOSE && !--x->unknown_level) + x->state = S_INDIR; + return 0; + } + + g_set_error(err, 1, 0, "Unexpected token in state %s: %s, %s", + x->state == S_START ? "START" : + x->state == S_FLOPEN ? "FLOPEN" : + x->state == S_DIROPEN ? "DIROPEN" : + x->state == S_INDIR ? "INDIR" : + x->state == S_FILEOPEN ? "FILEOPEN" : + x->state == S_INFILE ? "INFILE" : + x->state == S_END ? "END" : "UNKNOWN", + type == XMLT_OPEN ? "OPEN" : + type == XMLT_CLOSE ? "CLOSE" : + type == XMLT_ATTR ? "ATTR" : + type == XMLT_ATTDONE ? "ATTDONE" : "???", + arg1 ? arg1 : "<NULL>"); + return -1; +} + + +static int ctx_open(struct ctx *x, const char *file, GError **err) { + memset(x, 0, sizeof(struct ctx)); + + // open file + x->fh_f = fopen(file, "r"); + if(!x->fh_f) { + g_set_error_literal(err, 1, 0, g_strerror(errno)); + return -1; + } + + // open BZ2 decompression + if(strlen(file) > 4 && strcmp(file+(strlen(file)-4), ".bz2") == 0) { + int bzerr; + x->fh_bz = BZ2_bzReadOpen(&bzerr, x->fh_f, 0, 0, NULL, 0); + if(bzerr != BZ_OK) { + g_set_error(err, 1, 0, "Unable to open bzip2 file (%d): %s", bzerr, g_strerror(errno)); + return -1; + } + } + + return 0; +} + + +static void ctx_close(struct ctx *x) { + if(x->fh_bz) { + int bzerr; + BZ2_bzReadClose(&bzerr, x->fh_bz); + } + + if(x->fh_f) + fclose(x->fh_f); + + if(x->name) + g_free(x->name); +} + + +struct fl_list *fl_load(const char *file, GError **err, gboolean local) { + g_return_val_if_fail(err == NULL || *err == NULL, NULL); + + struct ctx x; + GError *ierr = NULL; + if(ctx_open(&x, file, &ierr)) + goto end; + + x.state = S_START; + x.root = fl_list_create("", FALSE); + x.root->sub = g_ptr_array_new_with_free_func(fl_list_free); + x.cur = x.root; + x.filesize = G_MAXUINT64; + x.local = local; + + if(xml_parse(entitycb, readcb, &x, &ierr)) + goto end; + +end: + g_return_val_if_fail(ierr || x.state == S_END, NULL); + ctx_close(&x); + if(ierr) { + g_propagate_error(err, ierr); + if(x.root) + fl_list_free(x.root); + x.root = NULL; + } + return x.root; +} + + + + + +// Async version of fl_load(). Performs the load in a background thread. Only +// used for non-local filelists. + +struct async_dat { + char *file; + void (*cb)(struct fl_list *, GError *, void *); + void *dat; + GError *err; + struct fl_list *fl; +}; + + +static gboolean async_d(gpointer dat) { + struct async_dat *arg = dat; + arg->cb(arg->fl, arg->err, arg->dat); + g_free(arg->file); + g_slice_free(struct async_dat, arg); + return FALSE; +} + + +static void async_f(gpointer dat, gpointer udat) { + struct async_dat *arg = dat; + arg->fl = fl_load(arg->file, &arg->err, FALSE); + g_idle_add(async_d, arg); +} + + +// Ownership of both the file list and the error is passed to the callback +// function. +void fl_load_async(const char *file, void (*cb)(struct fl_list *, GError *, void *), void *dat) { + static GThreadPool *pool = NULL; + if(!pool) + pool = g_thread_pool_new(async_f, NULL, 2, FALSE, NULL); + struct async_dat *arg = g_slice_new0(struct async_dat); + arg->file = g_strdup(file); + arg->dat = dat; + arg->cb = cb; + g_thread_pool_push(pool, arg, NULL); +} + diff --git a/src/fl_util.c b/src/fl_util.c index 909a997..74d38e6 100644 --- a/src/fl_util.c +++ b/src/fl_util.c @@ -25,12 +25,7 @@ #include "ncdc.h" -#include <errno.h> -#include <stdio.h> -#include <unistd.h> -#include <libxml/xmlreader.h> -#include <libxml/xmlwriter.h> -#include <bzlib.h> +#include <string.h> #if INTERFACE @@ -476,304 +471,3 @@ gboolean fl_search_match_full(struct fl_list *fl, struct fl_search *s) { return r; } - - - - - - -// Internal structure used by fl_load() - -struct fl_load_context { - char *file; // some name, for debugging purposes - BZFILE *fh_bz; // if BZ2 compression is enabled (implies fh_h!=NULL) - FILE *fh_f; // if we're working with a file - GError **err; - gboolean stream_end; -}; - - - - -// Read filelist from an xml file - -// Tries to "fix" invalid byte UTF-8 sequences, as present in some FlylinkDC++ -// generated lists. -static void fl_load_fixinput(char *buf, int len) { - while(len > 1) { - if(*buf == 0x1d) - *buf = '?'; - buf++; - len--; - } -} - -static int fl_load_input(void *context, char *buf, int len) { - struct fl_load_context *xc = context; - if(xc->stream_end) - return 0; - int bzerr; - if(xc->fh_bz) { - int r = BZ2_bzRead(&bzerr, xc->fh_bz, buf, len); - if(bzerr != BZ_OK && bzerr != BZ_STREAM_END) { - g_set_error(xc->err, 1, 0, "bzip2 decompression error. (%d)", bzerr); - return -1; - } else { - xc->stream_end = bzerr == BZ_STREAM_END; - fl_load_fixinput(buf, r); - return r; - } - } else if(xc->fh_f) { - int r = fread(buf, 1, len, xc->fh_f); - if(r < 0) - g_set_error(xc->err, 1, 0, "Read error: %s", g_strerror(errno)); - fl_load_fixinput(buf, r); - xc->stream_end = r <= 0; - return r; - } else - g_return_val_if_reached(-1); -} - - -static int fl_load_close(void *context) { - struct fl_load_context *xc = context; - int bzerr; - if(xc->fh_bz) - BZ2_bzReadClose(&bzerr, xc->fh_bz); - fclose(xc->fh_f); - g_free(xc->file); - g_slice_free(struct fl_load_context, xc); - return 0; -} - - -static void fl_load_error(void *arg, const char *msg, xmlParserSeverities severity, xmlTextReaderLocatorPtr locator) { - struct fl_load_context *xc = arg; - if(severity == XML_PARSER_SEVERITY_VALIDITY_WARNING || severity == XML_PARSER_SEVERITY_WARNING) - g_warning("XML parse warning in %s line %d: %s", xc->file, xmlTextReaderLocatorLineNumber(locator), msg); - else if(xc->err && !*(xc->err)) - g_set_error(xc->err, 1, 0, "XML parse error on input line %d: %s", xmlTextReaderLocatorLineNumber(locator), msg); -} - - -static int fl_load_handle(xmlTextReaderPtr reader, gboolean *havefl, gboolean *newdir, gboolean *flo, struct fl_list **cur, gboolean local) { - struct fl_list *tmp; - char *attr[3]; - char name[50], *tmpname; - - if(!(tmpname = (char *)xmlTextReaderName(reader))) - return -1; - strncpy(name, tmpname, 50); - name[49] = 0; - free(tmpname); - - switch(xmlTextReaderNodeType(reader)) { - - case XML_READER_TYPE_ELEMENT: - // No open elements in a <File> allowed - if(*flo) - return -1; - // <FileListing ..> - // We ignore its attributes (for now) - if(strcmp(name, "FileListing") == 0) { - if(*havefl) - return -1; - *havefl = TRUE; - // <Directory ..> - } else if(strcmp(name, "Directory") == 0) { - if(!*havefl) - return -1; - if(!(attr[0] = (char *)xmlTextReaderGetAttribute(reader, (xmlChar *)"Name"))) - return -1; - attr[1] = (char *)xmlTextReaderGetAttribute(reader, (xmlChar *)"Incomplete"); - if(attr[1] && strcmp(attr[1], "0") != 0 && strcmp(attr[1], "1") != 0) { - free(attr[0]); - return -1; - } - tmp = fl_list_create(attr[0], FALSE); - tmp->isfile = FALSE; - tmp->sub = g_ptr_array_new_with_free_func(fl_list_free); - fl_list_add(*newdir ? *cur : (*cur)->parent, tmp, -1); - *cur = tmp; - *newdir = !xmlTextReaderIsEmptyElement(reader); - free(attr[0]); - free(attr[1]); - // <File .. /> - } else if(strcmp(name, "File") == 0) { - if(!*havefl) - return -1; - if(!(attr[0] = (char *)xmlTextReaderGetAttribute(reader, (xmlChar *)"Name"))) - return -1; - attr[1] = (char *)xmlTextReaderGetAttribute(reader, (xmlChar *)"Size"); - if(!attr[1] || strspn(attr[1], "0123456789") != strlen(attr[1])) { - free(attr[0]); - return -1; - } - attr[2] = (char *)xmlTextReaderGetAttribute(reader, (xmlChar *)"TTH"); - if(!attr[2] || !istth(attr[2])) { - free(attr[0]); - free(attr[1]); - return -1; - } - tmp = fl_list_create(attr[0], local); - tmp->isfile = TRUE; - tmp->size = g_ascii_strtoull(attr[1], NULL, 10); - tmp->hastth = TRUE; - base32_decode(attr[2], tmp->tth); - fl_list_add(*newdir ? *cur : (*cur)->parent, tmp, -1); - *cur = tmp; - *newdir = FALSE; - free(attr[0]); - free(attr[1]); - free(attr[2]); - if(!xmlTextReaderIsEmptyElement(reader)) - *flo = TRUE; - } - break; - - case XML_READER_TYPE_END_ELEMENT: - // </File> - if(strcmp(name, "File") == 0) - *flo = FALSE; - // </Directory> - else if(strcmp(name, "Directory") == 0) { - if(!*newdir) - *cur = (*cur)->parent; - else - *newdir = FALSE; - fl_list_sort(*cur); - // </FileListing> - } else if(strcmp(name, "FileListing") == 0) { - return 0; // stop reading - } - break; - } - return 1; -} - - -static xmlTextReaderPtr fl_load_open(const char *file, GError **err) { - gboolean isbz2 = strlen(file) > 4 && strcmp(file+(strlen(file)-4), ".bz2") == 0; - - // open file - FILE *f = fopen(file, "r"); - if(!f) { - g_set_error_literal(err, 1, 0, g_strerror(errno)); - return NULL; - } - - // open BZ2 decompression - BZFILE *bzf = NULL; - if(isbz2) { - int bzerr; - bzf = BZ2_bzReadOpen(&bzerr, f, 0, 0, NULL, 0); - if(bzerr != BZ_OK) { - g_set_error(err, 1, 0, "Unable to open BZ2 file (%d)", bzerr); - fclose(f); - return NULL; - } - } - - // create reader - struct fl_load_context *xc = g_slice_new0(struct fl_load_context); - xc->err = err; - xc->file = g_strdup(file); - xc->fh_f = f; - xc->fh_bz = bzf; - xmlTextReaderPtr reader = xmlReaderForIO(fl_load_input, fl_load_close, xc, NULL, NULL, XML_PARSE_NOENT); - - if(!reader) { - fl_load_close(xc); - if(err && !*err) - g_set_error_literal(err, 1, 0, "Failed to open file."); - return NULL; - } - - xmlTextReaderSetErrorHandler(reader, fl_load_error, xc); - return reader; -} - - -struct fl_list *fl_load(const char *file, GError **err, gboolean local) { - g_return_val_if_fail(err == NULL || *err == NULL, NULL); - - // open the file - xmlTextReaderPtr reader = fl_load_open(file, err); - if(!reader) - return NULL; - - // parse & read - struct fl_list *cur, *root; - gboolean havefl = FALSE, newdir = TRUE, flo = FALSE; - int ret; - - root = fl_list_create("", FALSE); - root->sub = g_ptr_array_new_with_free_func(fl_list_free); - cur = root; - - while((ret = xmlTextReaderRead(reader)) == 1) - if((ret = fl_load_handle(reader, &havefl, &newdir, &flo, &cur, local)) <= 0) - break; - - if(ret < 0 || !havefl) { - if(err && !*err) // rather uninformative error message as fallback - g_set_error_literal(err, 1, 0, !havefl ? "No <FileListing> tag found." : "Error parsing or validating XML."); - fl_list_free(root); - root = NULL; - } else - fl_list_sort(root); - - // close (ignoring errors) - xmlTextReaderClose(reader); - xmlFreeTextReader(reader); - - return root; -} - - - - - -// Async version of fl_load(). Performs the load in a background thread. Only -// used for non-local filelists. - -static GThreadPool *fl_load_pool = NULL; - -struct fl_load_async_dat { - char *file; - void (*cb)(struct fl_list *, GError *, void *); - void *dat; - GError *err; - struct fl_list *fl; -}; - - -static gboolean fl_load_async_d(gpointer dat) { - struct fl_load_async_dat *arg = dat; - arg->cb(arg->fl, arg->err, arg->dat); - g_free(arg->file); - g_slice_free(struct fl_load_async_dat, arg); - return FALSE; -} - - -static void fl_load_async_f(gpointer dat, gpointer udat) { - struct fl_load_async_dat *arg = dat; - arg->fl = fl_load(arg->file, &arg->err, FALSE); - g_idle_add(fl_load_async_d, arg); -} - - -// Ownership of both the file list and the error is passed to the callback -// function. -void fl_load_async(const char *file, void (*cb)(struct fl_list *, GError *, void *), void *dat) { - if(!fl_load_pool) - fl_load_pool = g_thread_pool_new(fl_load_async_f, NULL, 2, FALSE, NULL); - struct fl_load_async_dat *arg = g_slice_new0(struct fl_load_async_dat); - arg->file = g_strdup(file); - arg->dat = dat; - arg->cb = cb; - g_thread_pool_push(fl_load_pool, arg, NULL); -} - - @@ -76,6 +76,7 @@ struct var; #include "commands.h" #include "db.h" #include "dl.h" +#include "fl_load.h" #include "fl_local.h" #include "fl_save.h" #include "fl_util.h" |