Skip to content

Commit

Permalink
ALlow standard input
Browse files Browse the repository at this point in the history
  • Loading branch information
@wolfmanx @avih
wolfmanx authored and avih committed Nov 19, 2017
1 parent bfcee65 commit c4305b8
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 52 deletions.
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ repository as of 2016-05-12 (as currently used by Firefox) [1].
Usage: dejsonlz4 [-h] IN_FILE [OUT_FILE]
-h Display this help and exit.
Decompress Mozilla bookmarks backup file IN_FILE to OUT_FILE.
If IN_FILE is '-', decompress from standard input.
If OUT_FILE is '-' or missing, decompress to standard output.
Note: IN_FILE is transferred to memory entirely before decompressing.
Decompression is also done in memory entirely before output.
```

## Build:
Expand Down
75 changes: 44 additions & 31 deletions src/dejsonlz4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,34 +52,56 @@ void exit_usage(int code) {
"Usage: dejsonlz4 [-h] IN_FILE [OUT_FILE]\n"
" -h Display this help and exit.\n"
"Decompress Mozilla bookmarks backup file IN_FILE to OUT_FILE.\n"
"If IN_FILE is '-', decompress from standard input.\n"
"If OUT_FILE is '-' or missing, decompress to standard output.\n"
"Note: IN_FILE is transferred to memory entirely before decompressing.\n"
"Decompression is also done in memory entirely before output.\n"
);
exit(code);
}

/* If required, prevents EOL translations with f. Returns non-zero on failure */
int ensure_binary(FILE *f)
{
#ifdef _WIN32
/* -1 is failure: https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx */
return _setmode(_fileno(f), O_BINARY) == -1;
#else
return 0; /* not required */
#endif
}

#define INITIAL_ALLOC_SIZE (32 * 1024)

/* if fname is NULL, reads from stdin till EOF */
void *file_to_mem(const char *fname, size_t *out_size)
{
size_t fsize = 0;
void *rv = 0;
FILE *f = fopen(fname, "rb");
unsigned char *buf = 0, *rv = 0;
size_t buf_size = 0, got = 0;
FILE *f = fname ? fopen(fname, "rb") : stdin;
if (!f)
goto cleanup;
if (fseek(f, 0, SEEK_END) < 0)
goto cleanup;
fsize = ftell(f);
fseek(f, 0, SEEK_SET);
if (!(rv = malloc(fsize)))
goto cleanup;
if (fsize != fread(rv, 1, fsize, f)) {
free(rv);
return 0; /* can't do anything */
if (!fname && ensure_binary(f))
fprintf(stderr, "Warning: cannot set stdin to binary mode\n");

do {
buf_size = got ? got * 2 : INITIAL_ALLOC_SIZE;
if (buf_size <= got || !(buf = realloc(buf, buf_size)))
break; /* size_t wrap-around or OOM: break before EOF */
rv = buf;
got += fread(buf + got, 1, buf_size - got, f);
} while (got == buf_size); /* otherwise feof(f) or ferror(f) */

if (!buf || !feof(f) || ferror(f)) {
if (rv)
free(rv);
rv = 0;
}

cleanup:
if (f)
if (f && fname)
fclose(f);
if (rv && out_size)
*out_size = fsize;
*out_size = got;
return rv;
}

Expand All @@ -98,13 +120,14 @@ int main(int argc, char **argv)
/* process arguments */
if ((argc > 3) || (argc < 2) || (argc > 1 && !strcmp(argv[1], "-h")))
exit_usage(argc == 2 ? 0 : 1);
iname = argv[1];
if (argc > 1 && strcmp("-", argv[1]))
iname = argv[1];
if (argc > 2 && strcmp("-", argv[2]))
oname = argv[2];

/* read input file and validate magic header and minimum size */
if (!(idata = file_to_mem(iname, &isize)))
ERR_CLEANUP("cannot read file '%s'\n", iname);
ERR_CLEANUP("cannot read file '%s'\n", iname ? iname : "<stdin>");
if (isize < magic_size + decomp_size || memcmp(mozlz4_magic, idata, magic_size))
ERR_CLEANUP("unsupported file format\n");

Expand All @@ -121,20 +144,10 @@ int main(int argc, char **argv)
fprintf(stderr, "Warning: decompressed file smaller than expected\n");

/* write output */
if (oname) {
if (!(ofile = fopen(oname, "wb")))
ERR_CLEANUP("cannot open '%s' for writing\n", oname);
} else {
#ifdef _WIN32
/* Try changing stdout to binary mode, or else it messes with EOL chars.
* Might not matter much for Mozilla files since they're text.
*/
if (_setmode(_fileno(stdout), O_BINARY) == -1)
fprintf(stderr, "Warning: cannot set output to binary mode\n");
#endif
ofile = stdout;
}

if (!(ofile = oname ? fopen(oname, "wb") : stdout))
ERR_CLEANUP("cannot open '%s' for writing\n", oname);
if (!oname && ensure_binary(ofile))
fprintf(stderr, "Warning: cannot set stdout to binary mode\n");
if (dsize != fwrite(odata, 1, dsize, ofile))
ERR_CLEANUP("cannot write to '%s'\n", oname ? oname : "<stdout>");

Expand Down
69 changes: 48 additions & 21 deletions src/ref_compress/jsonlz4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,34 +54,57 @@ void exit_usage(int code) {
"Usage: jsonlz4 [-h] IN_FILE OUT_FILE\n"
" -h Display this help and exit.\n"
"Compress IN_FILE to OUT_FILE with same format as Firefox bookmarks backup.\n"
"If IN_FILE is '-', compress from standard input.\n"
"If OUT_FILE is '-', compress to standard output.\n"
"Note: IN_FILE is transferred to memory entirely before compressing.\n"
"Compression is also done in memory entirely before output.\n"
"Note: it's not recommended to use this tool, as it creates non standard files.\n"
);
exit(code);
}

/* If required, prevents EOL translations with f. Returns non-zero on failure */
int ensure_binary(FILE *f)
{
#ifdef _WIN32
/* -1 is failure: https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx */
return _setmode(_fileno(f), O_BINARY) == -1;
#else
return 0; /* not required */
#endif
}

#define INITIAL_ALLOC_SIZE (32 * 1024)

/* if fname is NULL, reads from stdin till EOF */
void *file_to_mem(const char *fname, size_t *out_size)
{
size_t fsize = 0;
void *rv = 0;
FILE *f = fopen(fname, "rb");
unsigned char *buf = 0, *rv = 0;
size_t buf_size = 0, got = 0;
FILE *f = fname ? fopen(fname, "rb") : stdin;
if (!f)
goto cleanup;
if (fseek(f, 0, SEEK_END) < 0)
goto cleanup;
fsize = ftell(f);
fseek(f, 0, SEEK_SET);
if (!(rv = malloc(fsize)))
goto cleanup;
if (fsize != fread(rv, 1, fsize, f)) {
free(rv);
return 0; /* can't do anything */
if (!fname && ensure_binary(f))
fprintf(stderr, "Warning: cannot set stdin to binary mode\n");

do {
buf_size = got ? got * 2 : INITIAL_ALLOC_SIZE;
if (buf_size <= got || !(buf = realloc(buf, buf_size)))
break; /* size_t wrap-around or OOM: break before EOF */
rv = buf;
got += fread(buf + got, 1, buf_size - got, f);
} while (got == buf_size); /* otherwise feof(f) or ferror(f) */

if (!buf || !feof(f) || ferror(f)) {
if (rv)
free(rv);
rv = 0;
}

cleanup:
if (f)
if (f && fname)
fclose(f);
if (rv && out_size)
*out_size = fsize;
*out_size = got;
return rv;
}

Expand All @@ -100,12 +123,14 @@ int main(int argc, char **argv)
/* process arguments */
if (argc != 3)
exit_usage(argc == 2 && !strcmp(argv[1], "-h") ? 0 : 1);
iname = argv[1];
oname = argv[2];
if (strcmp("-", argv[1]))
iname = argv[1];
if (strcmp("-", argv[2]))
oname = argv[2];

/* read input file and allocate buffer for output file */
if (!(idata = file_to_mem(iname, &isize)))
ERR_CLEANUP("cannot read file '%s'\n", iname);
ERR_CLEANUP("cannot read file '%s'\n", iname ? iname : "<stdin>");
if (!(odata = malloc(magic_size + decomp_size + LZ4_compressBound(isize))))
ERR_CLEANUP("cannot allocate memory for output\n");

Expand All @@ -120,15 +145,17 @@ int main(int argc, char **argv)
osize = csize + magic_size + decomp_size;

/* write output */
if (!(ofile = fopen(oname, "wb")))
if (!(ofile = oname ? fopen(oname, "wb") : stdout))
ERR_CLEANUP("cannot open '%s' for writing\n", oname);
if (!oname && ensure_binary(ofile))
fprintf(stderr, "Warning: cannot set stdout to binary mode\n");
if (osize != fwrite(odata, 1, osize, ofile))
ERR_CLEANUP("cannot write to '%s'\n", oname);
ERR_CLEANUP("cannot write to '%s'\n", oname ? oname : "<stdout>");

rv = 0;

cleanup:
if (ofile)
if (ofile && oname)
fclose(ofile);
if (odata)
free(odata);
Expand Down

0 comments on commit c4305b8

Please sign in to comment.