summaryrefslogtreecommitdiff
path: root/src/fl_load.c
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2016-12-16 20:22:28 +0100
committerYorhel <git@yorhel.nl>2016-12-16 20:30:23 +0100
commite3c5ff32f4f16ba6862ed9e78dc59c1970a728ab (patch)
treed287c8dbedb4dc03e057ec53f8182fbc7dcc331f /src/fl_load.c
parentf53be2b1e3d27a9f22ebab1aa6b466da5c8de8dd (diff)
fl_load: Add support for reading multistream bzip2 filelists
This is a feature that FlylinkDC++ is experimenting with. This implementation rewrites the file reader to use unbuffered I/O calls, because the reader itself already does enough buffering. The BZ2 reading is rewritten to use the lower-level bzDecompress rather than bzRead API, because the latter seems harder to use w.r.t. multistream bzip2 files. It's slightly disappointing that the "high-level" bzRead API doesn't handle multistream files automatically when the bzcat utility does. Regardless of whether multistream file lists will actually be used, one would expect ncdc to be able to read any bzip2 file that the bzcat utility can read, hence this commit.
Diffstat (limited to 'src/fl_load.c')
-rw-r--r--src/fl_load.c86
1 files changed, 55 insertions, 31 deletions
diff --git a/src/fl_load.c b/src/fl_load.c
index a986369..92ec862 100644
--- a/src/fl_load.c
+++ b/src/fl_load.c
@@ -205,7 +205,37 @@ static void fl_load_token(ctx_t *x, yxml_ret_t r, GError **err) {
}
-static fl_list_t *fl_load_parse(FILE *fh, BZFILE *bzfh, gboolean local, GError **err) {
+static int fl_load_readbz(bz_stream *bzs, int fd, char *bzbuf, GError **err) {
+ int buflen;
+
+ bzs->next_in = bzbuf;
+ if(bzs->avail_in == 0) {
+ buflen = read(fd, bzs->next_in + bzs->avail_in, READBUFSIZE - bzs->avail_in);
+ if(buflen == 0)
+ return -1;
+ if(buflen < 0) {
+ g_set_error(err, 1, 0, "Read error: %s", g_strerror(errno));
+ return -1;
+ }
+ bzs->avail_in += buflen;
+ }
+
+ int bzerr = BZ2_bzDecompress(bzs);
+ if(bzerr == BZ_STREAM_END) {
+ BZ2_bzDecompressEnd(bzs);
+ BZ2_bzDecompressInit(bzs, 0, 0);
+ } else if(bzerr != BZ_OK) {
+ g_set_error(err, 1, 0, "bzip2 decompression error (%d): %s", bzerr, g_strerror(errno));
+ return -1;
+ }
+
+ memmove(bzbuf, bzs->next_in, bzs->avail_in);
+ bzs->next_in = bzbuf;
+ return READBUFSIZE-bzs->avail_out;
+}
+
+
+static fl_list_t *fl_load_parse(int fd, bz_stream *bzs, gboolean local, GError **err) {
ctx_t *x = g_new(ctx_t, 1);
x->state = S_START;
x->root = fl_list_create("", FALSE);
@@ -219,24 +249,21 @@ static fl_list_t *fl_load_parse(FILE *fh, BZFILE *bzfh, gboolean local, GError *
yxml_init(&x->x, x->stack, STACKSIZE);
int buflen = 0;
- int bzeof = 0;
+ char *bzbuf = NULL;
while(1) {
// Fill buffer
- if(bzfh) {
- if(bzeof)
- break;
- int bzerr;
- buflen = BZ2_bzRead(&bzerr, bzfh, x->buf, READBUFSIZE);
- if(bzerr == BZ_STREAM_END)
- bzeof = 1;
- else if(bzerr != BZ_OK) {
- g_set_error(err, 1, 0, "bzip2 decompression error (%d): %s", bzerr, g_strerror(errno));
+ if(bzs) {
+ if(!bzbuf)
+ bzbuf = g_malloc(READBUFSIZE);
+ bzs->next_out = x->buf;
+ bzs->avail_out = READBUFSIZE;
+ buflen = fl_load_readbz(bzs, fd, bzbuf, err);
+ if(buflen < 0)
break;
- }
} else {
- buflen = fread(x->buf, 1, READBUFSIZE, fh);
- if(buflen < 0 && feof(fh))
+ buflen = read(fd, x->buf, READBUFSIZE);
+ if(buflen == 0)
break;
if(buflen < 0) {
g_set_error(err, 1, 0, "Read error: %s", g_strerror(errno));
@@ -268,6 +295,7 @@ static fl_list_t *fl_load_parse(FILE *fh, BZFILE *bzfh, gboolean local, GError *
g_set_error_literal(err, 1, 0, "XML document did not end correctly");
fl_list_t *root = x->root;
+ g_free(bzbuf);
g_free(x->name);
g_free(x);
return root;
@@ -278,36 +306,32 @@ fl_list_t *fl_load(const char *file, GError **err, gboolean local) {
g_return_val_if_fail(err == NULL || *err == NULL, NULL);
fl_list_t *root = NULL;
- FILE *fh;
- BZFILE *bzfh = NULL;
+ int fd;
+ bz_stream *bzs = NULL;
GError *ierr = NULL;
// open file
- fh = fopen(file, "r");
- if(!fh) {
+ fd = open(file, O_RDONLY);
+ if(fd < 0) {
g_set_error_literal(&ierr, 1, 0, g_strerror(errno));
goto end;
}
- // open BZ2 decompression
+ // Create BZ2 stream object if this is a bzip2 file
if(strlen(file) > 4 && strcmp(file+(strlen(file)-4), ".bz2") == 0) {
- int bzerr;
- bzfh = BZ2_bzReadOpen(&bzerr, fh, 0, 0, NULL, 0);
- if(bzerr != BZ_OK) {
- g_set_error(&ierr, 1, 0, "Unable to open bzip2 file (%d): %s", bzerr, g_strerror(errno));
- goto end;
- }
+ bzs = g_new0(bz_stream, 1);
+ BZ2_bzDecompressInit(bzs, 0, 0);
}
- root = fl_load_parse(fh, bzfh, local, &ierr);
+ root = fl_load_parse(fd, bzs, local, &ierr);
end:
- if(bzfh) {
- int bzerr;
- BZ2_bzReadClose(&bzerr, bzfh);
+ if(bzs) {
+ BZ2_bzDecompressEnd(bzs);
+ g_free(bzs);
}
- if(fh)
- fclose(fh);
+ if(fd >= 0)
+ close(fd);
if(ierr) {
g_propagate_error(err, ierr);
if(root)