/* Copyright 2014 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* Example main() function for Brotli library. */ #include #include #include #include #include #include #include #include #include "../dec/decode.h" #include "../enc/encode.h" #if !defined(_WIN32) #include #else #include #include #define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO)) #if !defined(__MINGW32__) #define STDIN_FILENO MAKE_BINARY(_fileno(stdin)) #define STDOUT_FILENO MAKE_BINARY(_fileno(stdout)) #define S_IRUSR S_IREAD #define S_IWUSR S_IWRITE #endif #define fdopen _fdopen #define unlink _unlink #define fopen ms_fopen #define open ms_open #if defined(_MSC_VER) && (_MSC_VER >= 1400) #define fseek _fseeki64 #define ftell _ftelli64 #endif static FILE* ms_fopen(const char *filename, const char *mode) { FILE* result = 0; fopen_s(&result, filename, mode); return result; } static int ms_open(const char *filename, int oflag, int pmode) { int result = -1; _sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode); return result; } #endif /* WIN32 */ static int ParseQuality(const char* s, int* quality) { if (s[0] >= '0' && s[0] <= '9') { *quality = s[0] - '0'; if (s[1] >= '0' && s[1] <= '9') { *quality = *quality * 10 + s[1] - '0'; return (s[2] == 0) ? 1 : 0; } return (s[1] == 0) ? 1 : 0; } return 0; } #define UTILITY_NAME "Brotli" #define UTILITY_MAJOR_VERSION 0 #define UTILITY_MINOR_VERSION 5 #define UTILITY_REVERSION 2 static void ParseArgv(int argc, char **argv, char **input_path, char **output_path, char **dictionary_path, int *force, int *quality, int *gapmem, int *decompress, int *repeat, int *verbose, int *lgwin) { int k; *force = 0; *input_path = 0; *output_path = 0; *repeat = 1; *verbose = 0; *lgwin = 22; { size_t argv0_len = strlen(argv[0]); *decompress = argv0_len >= 5 && strcmp(&argv[0][argv0_len - 5], "unbro") == 0; } for (k = 1; k < argc; ++k) { if (!strcmp("--force", argv[k]) || !strcmp("-f", argv[k])) { if (*force != 0) { goto error; } *force = 1; continue; } else if (!strcmp("--decompress", argv[k]) || !strcmp("--uncompress", argv[k]) || !strcmp("-d", argv[k])) { *decompress = 1; continue; } else if (!strcmp("--verbose", argv[k]) || !strcmp("-v", argv[k])) { if (*verbose != 0) { goto error; } *verbose = 1; continue; } else if (!strcmp("--version", argv[k])) { fprintf(stderr, "%s Version %d.%d.%d %s\n", UTILITY_NAME, UTILITY_MAJOR_VERSION, UTILITY_MINOR_VERSION, UTILITY_REVERSION, __BUILD_VERSION); exit(1); } if (k < argc - 1) { if (!strcmp("--input", argv[k]) || !strcmp("--in", argv[k]) || !strcmp("-i", argv[k])) { if (*input_path != 0) { goto error; } *input_path = argv[k + 1]; ++k; continue; } else if (!strcmp("--output", argv[k]) || !strcmp("--out", argv[k]) || !strcmp("-o", argv[k])) { if (*output_path != 0) { goto error; } *output_path = argv[k + 1]; ++k; continue; } else if (!strcmp("--custom-dictionary", argv[k])) { if (*dictionary_path != 0) { goto error; } *dictionary_path = argv[k + 1]; ++k; continue; } else if (!strcmp("--quality", argv[k]) || !strcmp("-q", argv[k])) { if (!ParseQuality(argv[k + 1], quality)) { goto error; } ++k; continue; } else if (!strcmp("--repeat", argv[k]) || !strcmp("-r", argv[k])) { if (!ParseQuality(argv[k + 1], repeat)) { goto error; } ++k; continue; } else if (!strcmp("--window", argv[k]) || !strcmp("-w", argv[k])) { if (!ParseQuality(argv[k + 1], lgwin)) { goto error; } if (*lgwin < 10 || *lgwin >= 25) { goto error; } ++k; continue; } else if (!strcmp("--gap", argv[k]) || !strcmp("-g", argv[k])) { if (!ParseQuality(argv[k + 1], gapmem)) { goto error; } ++k; continue; } } goto error; } return; error: fprintf(stderr, "Usage: %s [--force] [--quality n] [--gap n] [--decompress]" " [--input filename] [--output filename] [--repeat iters]" " [--verbose] [--window n] [--custom-dictionary filename]" " [--version]\n", argv[0]); exit(1); } static FILE* OpenInputFile(const char* input_path) { FILE* f; if (input_path == 0) { return fdopen(STDIN_FILENO, "rb"); } f = fopen(input_path, "rb"); if (f == 0) { perror("fopen"); exit(1); } return f; } static FILE *OpenOutputFile(const char *output_path, const int force) { int fd; if (output_path == 0) { return fdopen(STDOUT_FILENO, "wb"); } fd = open(output_path, O_CREAT | (force ? 0 : O_EXCL) | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR); if (fd < 0) { if (!force) { struct stat statbuf; if (stat(output_path, &statbuf) == 0) { fprintf(stderr, "output file exists\n"); exit(1); } } perror("open"); exit(1); } return fdopen(fd, "wb"); } static int64_t FileSize(const char *path) { FILE *f = fopen(path, "rb"); int64_t retval; if (f == NULL) { return -1; } if (fseek(f, 0L, SEEK_END) != 0) { fclose(f); return -1; } retval = ftell(f); if (fclose(f) != 0) { return -1; } return retval; } /* Brotli specified memory allocate function */ static void *BrAlloc (void *memsize, size_t size) { *(int64_t *)memsize += size; return malloc(size); } /* Brotli specified memory free function */ static void BrFree (void *memsize, void *ptr) { free(ptr); } /* Result ownersip is passed to caller. |*dictionary_size| is set to resulting buffer size. */ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) { static const int kMaxDictionarySize = (1 << 24) - 16; FILE *f = fopen(path, "rb"); int64_t file_size_64; uint8_t* buffer; size_t bytes_read; if (f == NULL) { perror("fopen"); exit(1); } file_size_64 = FileSize(path); if (file_size_64 == -1) { fprintf(stderr, "could not get size of dictionary file"); exit(1); } if (file_size_64 > kMaxDictionarySize) { fprintf(stderr, "dictionary is larger than maximum allowed: %d\n", kMaxDictionarySize); exit(1); } *dictionary_size = (size_t)file_size_64; buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size); if (!buffer) { fprintf(stderr, "could not read dictionary: out of memory\n"); exit(1); } bytes_read = fread(buffer, sizeof(uint8_t), *dictionary_size, f); if (bytes_read != *dictionary_size) { fprintf(stderr, "could not read dictionary\n"); exit(1); } fclose(f); return buffer; } static const size_t kFileBufferSize = 65536; static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) { /* Dictionary should be kept during first rounds of decompression. */ uint8_t* dictionary = NULL; uint8_t* input; uint8_t* output; size_t total_out; size_t available_in; const uint8_t* next_in; size_t available_out = kFileBufferSize; uint8_t* next_out; BrotliResult result = BROTLI_RESULT_ERROR; BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize); if (!s) { fprintf(stderr, "out of memory\n"); return 0; } if (dictionary_path != NULL) { size_t dictionary_size = 0; dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize); BrotliSetCustomDictionary(dictionary_size, dictionary, s); } input = (uint8_t*)BrAlloc(memsize, kFileBufferSize); output = (uint8_t*)BrAlloc(memsize, kFileBufferSize); if (!input || !output) { fprintf(stderr, "out of memory\n"); goto end; } next_out = output; result = BROTLI_RESULT_NEEDS_MORE_INPUT; while (1) { if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) { if (feof(fin)) { break; } available_in = fread(input, 1, kFileBufferSize, fin); next_in = input; if (ferror(fin)) { break; } } else if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) { fwrite(output, 1, kFileBufferSize, fout); if (ferror(fout)) { break; } available_out = kFileBufferSize; next_out = output; } else { break; /* Error or success. */ } result = BrotliDecompressStream(&available_in, &next_in, &available_out, &next_out, &total_out, s); } if (next_out != output) { fwrite(output, 1, (size_t)(next_out - output), fout); } if ((result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) || ferror(fout)) { fprintf(stderr, "failed to write output\n"); } else if (result != BROTLI_RESULT_SUCCESS) { /* Error or needs more input. */ fprintf(stderr, "corrupt input\n"); } end: BrFree(memsize, dictionary); BrFree(memsize, input); BrFree(memsize, output); BrotliDestroyState(s); return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0; } static int Compress(int quality, int lgwin, FILE* fin, FILE* fout, const char *dictionary_path, void *memsize) { BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize); uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1); uint8_t* input = buffer; uint8_t* output = buffer + kFileBufferSize; size_t available_in = 0; const uint8_t* next_in = NULL; size_t available_out = kFileBufferSize; uint8_t* next_out = output; int is_eof = 0; int is_ok = 1; if (!s || !buffer) { is_ok = 0; goto finish; } BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality); BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin); if (dictionary_path != NULL) { size_t dictionary_size = 0; uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize); BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary); BrFree(memsize, dictionary); } while (1) { if (available_in == 0 && !is_eof) { available_in = fread(input, 1, kFileBufferSize, fin); next_in = input; if (ferror(fin)) break; is_eof = feof(fin); } if (!BrotliEncoderCompressStream(s, is_eof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS, &available_in, &next_in, &available_out, &next_out, NULL)) { is_ok = 0; break; } if (available_out != kFileBufferSize) { size_t out_size = kFileBufferSize - available_out; fwrite(output, 1, out_size, fout); if (ferror(fout)) break; available_out = kFileBufferSize; next_out = output; } if (BrotliEncoderIsFinished(s)) break; } finish: BrFree(memsize, buffer); BrotliEncoderDestroyInstance(s); if (!is_ok) { /* Should detect OOM? */ fprintf(stderr, "failed to compress data\n"); return 0; } else if (ferror(fout)) { fprintf(stderr, "failed to write output\n"); return 0; } else if (ferror(fin)) { fprintf(stderr, "failed to read input\n"); return 0; } return 1; } #define GAP_MEM_BLOCK 4096 int main(int argc, char** argv) { char *input_path = 0; char *output_path = 0; char *dictionary_path = 0; int force = 0; int quality = 11; int gmem = 1; int decompress = 0; int repeat = 1; int verbose = 0; int lgwin = 0; clock_t clock_start; int i; int64_t originsize = 0; int64_t msize = 0; ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force, &quality, &gmem, &decompress, &repeat, &verbose, &lgwin); clock_start = clock(); for (i = 0; i < repeat; ++i) { FILE* fin = OpenInputFile(input_path); FILE* fout = OpenOutputFile(output_path, force || repeat); int is_ok = 0; if (decompress) { if (fseek(fin, 16, SEEK_SET) != 0) { fclose(fin); return -1; } is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize); } else { originsize = FileSize(input_path); /* get original file size */ fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */ fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */ is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize); } if (!is_ok) { unlink(output_path); exit(1); } if (fclose(fin) != 0) { perror("fclose"); exit(1); } if (fclose(fout) != 0) { perror("fclose"); exit(1); } /* after compression operation then execute decompression operation to get decompression required memory size. */ if (decompress == 0) { fin = OpenInputFile(output_path); fout = tmpfile (); msize = 0; if (fseek(fin, 16, SEEK_SET) != 0) { fclose(fin); return -1; } is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize); if (!is_ok) { exit(1); } if (fclose(fin) != 0) { perror("fclose"); exit(1); } if (fclose(fout) != 0) { perror("fclose"); exit(1); } fout = fopen(output_path, "rb+"); /* open output_path file and add in head info */ /* seek to the offset of decompression required memory size */ if (fseek(fout, 8, SEEK_SET) != 0) { fclose(fout); return -1; } msize += gmem * GAP_MEM_BLOCK; /* there is a memory gap between IA32 and X64 environment*/ fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */ if (fclose(fout) != 0) { perror("fclose"); exit(1); } } } if (verbose) { clock_t clock_end = clock(); double duration = (double)(clock_end - clock_start) / CLOCKS_PER_SEC; int64_t uncompressed_size; double uncompressed_bytes_in_MB; if (duration < 1e-9) { duration = 1e-9; } uncompressed_size = FileSize(decompress ? output_path : input_path); if (uncompressed_size == -1) { fprintf(stderr, "failed to determine uncompressed file size\n"); exit(1); } uncompressed_bytes_in_MB = (double)(repeat * uncompressed_size) / (1024.0 * 1024.0); if (decompress) { printf("Brotli decompression speed: "); } else { printf("Brotli compression speed: "); } printf("%g MB/s\n", uncompressed_bytes_in_MB / duration); } return 0; }