BaseTools: Add Brotli algorithm tool

- Add Brotli algorithm tool support

Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Bell Song <binx.song@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

BaseTools/BinWrappers/PosixLike/Brotli

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinWrappers/PosixLike/BrotliCompress

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# This script will exec Brotli tool.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution.  The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+LVL="--quality 9"
+
+while [ $# != 0 ];do
+  case $1 in
+    -d)
+      ARGS+="--decompress "
+    ;;
+    -e)
+    ;;
+    -g)
+      ARGS+="--gap $2 "
+      shift
+    ;;
+    -l)
+      LVL="--quality $2 "
+      shift
+    ;;
+    -o)
+      ARGS+="--output $2 "
+      shift
+    ;;
+    *)
+      ARGS+="--input $1 "
+  esac
+
+shift
+done
+
+exec Brotli $ARGS $LVL

BaseTools/Conf/tools_def.template

@@ -7706,6 +7706,12 @@ RELEASE_RVCTCYGWIN_ARM_CC_FLAGS  = "$(CCPATH_FLAG)" $(ARCHCC_FLAGS) $(PLATFORM_F
 *_*_*_RSA2048SHA256SIGN_PATH = Rsa2048Sha256Sign
 *_*_*_RSA2048SHA256SIGN_GUID = A7717414-C616-4977-9420-844712A735BF
 
+##################
+# BrotliCompress tool definitions
+##################
+*_*_*_BROTLI_PATH        = BrotliCompress
+*_*_*_BROTLI_GUID        = 3D532050-5CDA-4FD0-879E-0F7F630D5AFB
+
 ##################
 # LzmaCompress tool definitions
 ##################

BaseTools/Source/C/BrotliCompress/BrotliCompress.bat

@@ -0,0 +1,48 @@
+@echo off
+@setlocal
+
+set LVL=--quality 9
+
+:Begin
+if "%1"=="" goto End
+
+if "%1"=="-d" (
+  set ARGS=%ARGS% --decompress
+  shift
+  goto Begin
+)
+
+if "%1"=="-e" (
+  shift
+  goto Begin
+)
+
+if "%1"=="-g" (
+  set ARGS=%ARGS% --gap %2
+  shift
+  shift
+  goto Begin
+)
+
+if "%1"=="-l" (
+  set LVL=--quality %2
+  shift
+  shift
+  goto Begin
+)
+
+if "%1"=="-o" (
+  set ARGS=%ARGS% --output %2
+  set INTMP=%2
+  shift
+  shift
+  goto Begin
+)
+
+set ARGS=%ARGS% --input %1
+shift
+goto Begin
+
+:End
+Brotli %ARGS% %LVL%
+@echo on

BaseTools/Source/C/BrotliCompress/GNUmakefile

@@ -0,0 +1,43 @@
+## @file
+# GNU/Linux makefile for 'Brotli' module build.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution.  The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+ARCH ?= IA32
+MAKEROOT ?= ..
+
+APPNAME = Brotli
+
+OBJECTS = \
+  tools/bro.o \
+  common/dictionary.o \
+  dec/bit_reader.o \
+  dec/decode.o \
+  dec/huffman.o \
+  dec/state.o \
+  enc/backward_references.o \
+  enc/bit_cost.o \
+  enc/block_splitter.o \
+  enc/brotli_bit_stream.o \
+  enc/cluster.o \
+  enc/compress_fragment.o \
+  enc/compress_fragment_two_pass.o \
+  enc/encode.o \
+  enc/entropy_encode.o \
+  enc/histogram.o \
+  enc/literal_cost.o \
+  enc/memory.o \
+  enc/metablock.o \
+  enc/static_dict.o \
+  enc/utf8_util.o
+
+include $(MAKEROOT)/Makefiles/app.makefile
+
+LIBS += -lm

BaseTools/Source/C/BrotliCompress/Makefile

@@ -0,0 +1,60 @@
+## @file
+# Windows makefile for 'Brotli' module build.
+#
+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution.  The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+!INCLUDE ..\Makefiles\ms.common
+
+CFLAGS = $(CFLAGS) /W2
+
+APPNAME = Brotli
+
+#LIBS = $(LIB_PATH)\Common.lib
+
+COMMON_OBJ = common\dictionary.obj
+DEC_OBJ = \
+  dec\bit_reader.obj \
+  dec\decode.obj \
+  dec\huffman.obj \
+  dec\state.obj
+ENC_OBJ = \
+  enc\backward_references.obj \
+  enc\bit_cost.obj \
+  enc\block_splitter.obj \
+  enc\brotli_bit_stream.obj \
+  enc\cluster.obj \
+  enc\compress_fragment.obj \
+  enc\compress_fragment_two_pass.obj \
+  enc\encode.obj \
+  enc\entropy_encode.obj \
+  enc\histogram.obj \
+  enc\literal_cost.obj \
+  enc\memory.obj \
+  enc\metablock.obj \
+  enc\static_dict.obj \
+  enc\utf8_util.obj
+
+OBJECTS = \
+  tools\bro.obj \
+  $(COMMON_OBJ) \
+  $(DEC_OBJ) \
+  $(ENC_OBJ)
+
+!INCLUDE ..\Makefiles\ms.app
+
+all: $(BIN_PATH)\BrotliCompress.bat
+
+$(BIN_PATH)\BrotliCompress.bat: BrotliCompress.bat
+  copy BrotliCompress.bat $(BIN_PATH)\BrotliCompress.bat /Y
+
+cleanall: localCleanall
+
+localCleanall:
+  del /f /q $(BIN_PATH)\BrotliCompress.bat > nul

BaseTools/Source/C/BrotliCompress/ReadMe.txt

@@ -0,0 +1,2 @@
+It is based on the Brotli v0.5.2.
+Brotli was released on the website https://github.com/google/brotli.

BaseTools/Source/C/BrotliCompress/tools/bro.c

@@ -73,6 +73,7 @@ static void ParseArgv(int argc, char **argv,
                       char **dictionary_path,
                       int *force,
                       int *quality,
+                      int *gapmem,
                       int *decompress,
                       int *repeat,
                       int *verbose,
@@ -160,6 +161,13 @@ static void ParseArgv(int argc, char **argv,
         }
         ++k;
         continue;
+      } else if (!strcmp("--gap", argv[k]) ||
+                  !strcmp("-g", argv[k])) {
+        if (!ParseQuality(argv[k + 1], gapmem)) {
+          goto error;
+        }
+        ++k;
+        continue;
       }
     }
     goto error;
@@ -167,7 +175,7 @@ static void ParseArgv(int argc, char **argv,
   return;
 error:
   fprintf(stderr,
-          "Usage: %s [--force] [--quality n] [--decompress]"
+          "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"
           " [--input filename] [--output filename] [--repeat iters]"
           " [--verbose] [--window n] [--custom-dictionary filename]\n",
           argv[0]);
@@ -225,9 +233,20 @@ static int64_t FileSize(const char *path) {
   return retval;
 }
 
+/* Brotli specified memory allocate function */
+static void *BrAlloc (void *memsize, size_t size) {
+  *(int64_t *)memsize += size;
+  return malloc(size);
+}
+
+/* Brotli specified memory free function */
+static void BrFree (void *memsize, void *ptr) {
+  free(ptr);
+}
+
 /* Result ownersip is passed to caller.
    |*dictionary_size| is set to resulting buffer size. */
-static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
+static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) {
   static const int kMaxDictionarySize = (1 << 24) - 16;
   FILE *f = fopen(path, "rb");
   int64_t file_size_64;
@@ -252,7 +271,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
   }
   *dictionary_size = (size_t)file_size_64;
 
-  buffer = (uint8_t*)malloc(*dictionary_size);
+  buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size);
   if (!buffer) {
     fprintf(stderr, "could not read dictionary: out of memory\n");
     exit(1);
@@ -268,7 +287,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
 
 static const size_t kFileBufferSize = 65536;
 
-static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
+static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) {
   /* Dictionary should be kept during first rounds of decompression. */
   uint8_t* dictionary = NULL;
   uint8_t* input;
@@ -279,18 +298,18 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
   size_t available_out = kFileBufferSize;
   uint8_t* next_out;
   BrotliResult result = BROTLI_RESULT_ERROR;
-  BrotliState* s = BrotliCreateState(NULL, NULL, NULL);
+  BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize);
   if (!s) {
     fprintf(stderr, "out of memory\n");
     return 0;
   }
   if (dictionary_path != NULL) {
     size_t dictionary_size = 0;
-    dictionary = ReadDictionary(dictionary_path, &dictionary_size);
+    dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
     BrotliSetCustomDictionary(dictionary_size, dictionary, s);
   }
-  input = (uint8_t*)malloc(kFileBufferSize);
+  input = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
-  output = (uint8_t*)malloc(kFileBufferSize);
+  output = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
   if (!input || !output) {
     fprintf(stderr, "out of memory\n");
     goto end;
@@ -331,17 +350,17 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
   }
 
 end:
-  free(dictionary);
+  BrFree(memsize, dictionary);
-  free(input);
+  BrFree(memsize, input);
-  free(output);
+  BrFree(memsize, output);
   BrotliDestroyState(s);
   return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0;
 }
 
 static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
-    const char *dictionary_path) {
+    const char *dictionary_path, void *memsize) {
-  BrotliEncoderState* s = BrotliEncoderCreateInstance(0, 0, 0);
+  BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize);
-  uint8_t* buffer = (uint8_t*)malloc(kFileBufferSize << 1);
+  uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1);
   uint8_t* input = buffer;
   uint8_t* output = buffer + kFileBufferSize;
   size_t available_in = 0;
@@ -360,9 +379,9 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
   BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
   if (dictionary_path != NULL) {
     size_t dictionary_size = 0;
-    uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size);
+    uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
     BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);
-    free(dictionary);
+    BrFree(memsize, dictionary);
   }
 
   while (1) {
@@ -392,7 +411,7 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
   }
 
 finish:
-  free(buffer);
+  BrFree(memsize, buffer);
   BrotliEncoderDestroyInstance(s);
 
   if (!is_ok) {
@@ -409,29 +428,41 @@ finish:
   return 1;
 }
 
+#define GAP_MEM_BLOCK  4096
+
 int main(int argc, char** argv) {
   char *input_path = 0;
   char *output_path = 0;
   char *dictionary_path = 0;
   int force = 0;
   int quality = 11;
+  int gmem = 1;
   int decompress = 0;
   int repeat = 1;
   int verbose = 0;
   int lgwin = 0;
   clock_t clock_start;
   int i;
+  int64_t originsize = 0;
+  int64_t msize = 0;
   ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,
-            &quality, &decompress, &repeat, &verbose, &lgwin);
+            &quality, &gmem, &decompress, &repeat, &verbose, &lgwin);
   clock_start = clock();
   for (i = 0; i < repeat; ++i) {
     FILE* fin = OpenInputFile(input_path);
     FILE* fout = OpenOutputFile(output_path, force || repeat);
     int is_ok = 0;
     if (decompress) {
-      is_ok = Decompress(fin, fout, dictionary_path);
+      if (fseek(fin, 16, SEEK_SET) != 0) {
+        fclose(fin);
+        return -1;
+      }
+      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
     } else {
-      is_ok = Compress(quality, lgwin, fin, fout, dictionary_path);
+      originsize = FileSize(input_path);  /* get original file size */
+      fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */
+      fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */
+      is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize);
     }
     if (!is_ok) {
       unlink(output_path);
@@ -445,6 +476,41 @@ int main(int argc, char** argv) {
       perror("fclose");
       exit(1);
     }
+    /* after compression operation then execute decompression operation
+       to get decompression required memory size. */
+    if (decompress == 0) {
+      fin = OpenInputFile(output_path);
+      fout = tmpfile ();
+      msize = 0;
+      if (fseek(fin, 16, SEEK_SET) != 0) {
+        fclose(fin);
+        return -1;
+      }
+      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
+      if (!is_ok) {
+        exit(1);
+      }
+      if (fclose(fin) != 0) {
+        perror("fclose");
+        exit(1);
+      }
+      if (fclose(fout) != 0) {
+        perror("fclose");
+        exit(1);
+      }
+      fout = fopen(output_path, "rb+");  /* open output_path file and add in head info */
+      /* seek to the offset of decompression required memory size */
+      if (fseek(fout, 8, SEEK_SET) != 0) {
+        fclose(fout);
+        return -1;
+      }
+      msize += gmem * GAP_MEM_BLOCK;  /* there is a memory gap between IA32 and X64 environment*/
+      fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */
+      if (fclose(fout) != 0) {
+        perror("fclose");
+        exit(1);
+      }
+    }
   }
   if (verbose) {
     clock_t clock_end = clock();

BaseTools/Source/C/GNUmakefile

@@ -52,6 +52,7 @@ LIBRARIES = Common
 APPLICATIONS = \
   GnuGenBootSector \
   BootSectImage \
+  BrotliCompress \
   EfiLdrImage \
   EfiRom \
   GenFfs \

BaseTools/Source/C/Makefile

@@ -17,6 +17,7 @@ ARCH = IA32
 LIBRARIES = Common
 APPLICATIONS = \
   BootSectImage \
+  BrotliCompress \
   EfiLdrImage \
   EfiRom \
   GenBootSector \