BaseTools: Use pip module if available, CI uses it by default

Use the new edk2-basetools pip module.
Includes a helpful message in setup to let users know which has been
selected.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Yuwei Chen <yuwei.chen@intel.com>

Reviewed-by: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Yuwei <Chen<yuwei.chen@intel.com>

Signed-off-by: Matthew Carlson <matthewfcarlson@gmail.com>

.pytool/CISettings.py

@@ -22,16 +22,24 @@ class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManag
         self.ActualTargets = []
         self.ActualArchitectures = []
         self.ActualToolChainTag = ""
+        self.UseBuiltInBaseTools = None
+        self.ActualScopes = None
 
     # ####################################################################################### #
     #                             Extra CmdLine configuration                                 #
     # ####################################################################################### #
 
     def AddCommandLineOptions(self, parserObj):
-        pass
+        group = parserObj.add_mutually_exclusive_group()
+        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")
+        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")
 
     def RetrieveCommandLineOptions(self, args):
-        pass
+        super().RetrieveCommandLineOptions(args)
+        if args.force_piptools:
+            self.UseBuiltInBaseTools = True
+        if args.no_piptools:
+            self.UseBuiltInBaseTools = False
 
     # ####################################################################################### #
     #                        Default Support for this Ci Build                                #
@@ -128,19 +136,38 @@ class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManag
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        scopes = ("cibuild", "edk2-build", "host-based-test")
+        if self.ActualScopes is None:
+            scopes = ("cibuild", "edk2-build", "host-based-test")
 
-        self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
+            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
 
-        if GetHostInfo().os.upper() == "LINUX" and self.ActualToolChainTag.upper().startswith("GCC"):
-            if "AARCH64" in self.ActualArchitectures:
-                scopes += ("gcc_aarch64_linux",)
-            if "ARM" in self.ActualArchitectures:
-                scopes += ("gcc_arm_linux",)
-            if "RISCV64" in self.ActualArchitectures:
-                scopes += ("gcc_riscv64_unknown",)
+            is_linux = GetHostInfo().os.upper() == "LINUX"
 
-        return scopes
+            if self.UseBuiltInBaseTools is None:
+                is_linux = GetHostInfo().os.upper() == "LINUX"
+                # try and import the pip module for basetools
+                try:
+                    import edk2basetools
+                    self.UseBuiltInBaseTools = True
+                except ImportError:
+                    self.UseBuiltInBaseTools = False
+                    pass
+
+            if self.UseBuiltInBaseTools == True:
+                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+                logging.warning("Using Pip Tools based BaseTools")
+            else:
+                logging.warning("Falling back to using in-tree BaseTools")
+
+            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):
+                if "AARCH64" in self.ActualArchitectures:
+                    scopes += ("gcc_aarch64_linux",)
+                if "ARM" in self.ActualArchitectures:
+                    scopes += ("gcc_arm_linux",)
+                if "RISCV64" in self.ActualArchitectures:
+                    scopes += ("gcc_riscv64_unknown",)
+            self.ActualScopes = scopes
+        return self.ActualScopes
 
     def GetRequiredSubmodules(self):
         ''' return iterable containing RequiredSubmodule objects.

BaseTools/BinPipWrappers/PosixLike/AmlToC

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+exe=$(basename "$full_cmd")
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" "$dir/../../Source/Python/$exe/$exe.py" "$@"

BaseTools/BinPipWrappers/PosixLike/BPDG

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.EccMain "$@"

BaseTools/BinPipWrappers/PosixLike/BrotliCompress

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# This script will exec Brotli tool with -e/-d options.
+#
+# Copyright (c) 2017 - 2018, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+QLT="-q 9 -w 22"
+ARGS=
+
+while test $# -gt 0
+do
+  case $1 in
+    -e)
+      ;;
+    -d)
+      ARGS+="$1 "
+      ;;
+    -o|-g)
+      ARGS+="$1 $2 "
+      shift
+      ;;
+    -q)
+      QLT="$1 $2 "
+      shift
+      ;;
+    *)
+      ARGS+="$1 "
+      ;;
+  esac
+  shift
+done
+
+exec Brotli $QLT $ARGS

BaseTools/BinPipWrappers/PosixLike/DevicePath

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/Ecc

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" -m $cmd.EccMain "$@"

BaseTools/BinPipWrappers/PosixLike/EfiRom

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenCrc32

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenDepex

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.AutoGen.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/GenFds

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/GenFfs

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenFv

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenFw

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenPatchPcdTable

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/GenSec

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/GenerateCapsule

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.Capsule.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/LzmaCompress

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/LzmaF86Compress

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# This script will exec LzmaCompress tool with --f86 option that enables converter for x86 code.
+#
+# (C) Copyright 2016 Hewlett Packard Enterprise Development LP<BR>
+# Copyright (c) 2012, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+for arg; do
+  case $arg in
+    -e|-d)
+      set -- "$@" --f86
+      break
+    ;;
+  esac
+done
+
+exec LzmaCompress "$@"

BaseTools/BinPipWrappers/PosixLike/PatchPcdValue

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/Pkcs7Sign

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/Rsa2048Sha256GenerateKeys

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.Rsa2048Sha256Sign.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/Rsa2048Sha256Sign

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/Split

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/TargetTool

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/TianoCompress

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/Trim

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=$(basename "$full_cmd")
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"
+

BaseTools/BinPipWrappers/PosixLike/UPT

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/VfrCompile

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/VolInfo

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+cmd=${full_cmd##*/}
+
+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
+then
+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
+then
+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
+  then
+    echo "BaseTools C Tool binary was not found ($cmd)"
+    echo "You may need to run:"
+    echo "  make -C $EDK_TOOLS_PATH/Source/C"
+  else
+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
+  fi
+elif [ -e "$dir/../../Source/C/bin/$cmd" ]
+then
+  exec "$dir/../../Source/C/bin/$cmd" "$@"
+else
+  echo "Unable to find the real '$cmd' to run"
+  echo "This message was printed by"
+  echo "  $0"
+  exit 127
+fi
+

BaseTools/BinPipWrappers/PosixLike/build

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+cmd=${full_cmd##*/}
+
+exec "${python_exe:-python}" -m edk2basetools.$cmd.$cmd "$@"

BaseTools/BinPipWrappers/PosixLike/posix_path_env.yaml

@@ -0,0 +1,11 @@
+## @file
+# Set this folder on the path for all linux builds
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "pipbuild-unix",
+  "override_id": "binwrappers",
+  "flags": ["set_path"]
+}

BaseTools/BinPipWrappers/WindowsLike/AmlToC.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/BPDG.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/Ecc.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.EccMain %*

BaseTools/BinPipWrappers/WindowsLike/GenDepex.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.AutoGen.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/GenFds.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/GenPatchPcdTable.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/GenerateCapsule.bat

@@ -0,0 +1 @@
+@%PYTHON_COMMAND% -m edk2basetools.Capsule.GenerateCapsule %*

BaseTools/BinPipWrappers/WindowsLike/PatchPcdValue.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/Pkcs7Sign.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/Rsa2048Sha256GenerateKeys.bat

@@ -0,0 +1 @@
+@%PYTHON_COMMAND% -m edk2basetools.Rsa2048Sha256Sign.Rsa2048Sha256GenerateKeys %*

BaseTools/BinPipWrappers/WindowsLike/Rsa2048Sha256Sign.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/Split.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/TargetTool.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/Trim.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/UPT.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/build.bat

@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% -m edk2basetools.%ToolName%.%ToolName% %*

BaseTools/BinPipWrappers/WindowsLike/win_build_tools_path_env.yaml

@@ -0,0 +1,11 @@
+## @file
+# Add this folder to the path on Windows
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "pipbuild-win",
+  "override_id": "binwrappers",
+  "flags": ["set_path"]
+}

BaseTools/BinWrappers/PosixLike/posix_path_env.yaml

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 {
+  "id": "binwrappers",
   "scope": "global-nix",
   "flags": ["set_path"]
 }

BaseTools/BinWrappers/WindowsLike/win_build_tools_path_env.yaml

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 {
+  "id": "binwrappers",
   "scope": "global-win",
   "flags": ["set_path"]
 }

BaseTools/BuildEnv

@@ -46,7 +46,7 @@ RestorePreviousConfiguration() {
       done
     fi
   fi
-  
+
   PREVIOUS_CONF_FILE=$CONF_PATH/BuildEnv.sh
   if [ -e $PREVIOUS_CONF_FILE ]
   then
@@ -198,7 +198,17 @@ AddEdkToolsToPath() {
 
   EDK_TOOLS_PATH_BIN=`GetEdkToolsPathBinDirectory`
 
-  AddDirToStartOfPath $EDK_TOOLS_PATH/BinWrappers/PosixLike
+  # check if the edk2basetools pip package is available
+  if $PYTHON_COMMAND -c "import edk2basetools" &> /dev/null; then
+    # if it is, use the pip version of the wrappers
+    echo "Using Pip Basetools"
+    AddDirToStartOfPath $EDK_TOOLS_PATH/BinPipWrappers/PosixLike
+  else
+    echo "Using EDK2 in-source Basetools"
+    AddDirToStartOfPath $EDK_TOOLS_PATH/BinWrappers/PosixLike
+  fi
+
+
   AddDirToStartOfPath $EDK_TOOLS_PATH_BIN
 
 }

BaseTools/Scripts/PatchCheck.py

@@ -359,6 +359,7 @@ class GitDiffCheck:
                 self.force_notabs = True
                 if self.filename.endswith('.sh') or \
                     self.filename.startswith('BaseTools/BinWrappers/PosixLike/') or \
+                    self.filename.startswith('BaseTools/BinPipWrappers/PosixLike/') or \
                     self.filename.startswith('BaseTools/Bin/CYGWIN_NT-5.1-i686/') or \
                     self.filename == 'BaseTools/BuildEnv':
                     #

BaseTools/Source/Python/README.md

@@ -0,0 +1,29 @@
+# Edk2 Basetools
+
+This folder has traditionally held the source of Python based tools used by EDK2.
+The official repo this source has moved to https://github.com/tianocore/edk2-basetools.
+This folder will remain in the tree until the next stable release (expected 202102).
+There is a new folder under Basetools `BinPipWrappers` that uses the pip module rather than this tree for Basetools.
+By adding the scope `pipbuild-win` or `pipbuild-unix` (depending on your host system), the SDE will use the
+`BinPipWrappers` instead of the regular `BinWrappers`.
+
+## Why Move It?
+
+The discussion is on the mailing list. The RFC is here: https://edk2.groups.io/g/rfc/topic/74009714#270
+The benefits allow for the Basetools project to be used separately from EDK2 itself as well as offering it in a
+globally accessible manner.
+This makes it much easier to build a module using Basetools.
+Separating the Basetools into their own repo allows for easier CI and contribution process.
+Additional pros, cons, and process can be found on the mailing list.
+
+## How Do I Install It?
+
+By default, EDK2 is tied to and tested with a specific version of the Basetools through `pip-requirements.txt`.
+You can simply run:
+
+```bash
+pip install -r pip-requirements.txt
+```
+
+This will install the required module, thought we strongly suggest setting up a virtual environment.
+Additionally, you can also install a local clone of the Basetools as well as a specific git commit.

BaseTools/toolsetup.bat

@@ -393,12 +393,31 @@ goto end
     goto check_freezer_path
   )
 
+
+
 :check_freezer_path
   endlocal
+
+  %PYTHON_COMMAND% -c "import edk2basetools" >NUL 2>NUL
+  if %ERRORLEVEL% EQU 0 (
+    goto use_pip_basetools
+  ) else (
+    goto use_builtin_basetools
+  )
+
+:use_builtin_basetools
+  @echo Using EDK2 in-source Basetools
   if defined BASETOOLS_PYTHON_SOURCE goto print_python_info
   set "PATH=%BASE_TOOLS_PATH%\BinWrappers\WindowsLike;%PATH%"
   set BASETOOLS_PYTHON_SOURCE=%BASE_TOOLS_PATH%\Source\Python
   set PYTHONPATH=%BASETOOLS_PYTHON_SOURCE%;%PYTHONPATH%
+  goto print_python_info
+
+:use_pip_basetools
+  @echo Using Pip Basetools
+  set "PATH=%BASE_TOOLS_PATH%\BinPipWrappers\WindowsLike;%PATH%"
+  set BASETOOLS_PYTHON_SOURCE=edk2basetools
+  goto print_python_info
 
 :print_python_info
   echo                PATH = %PATH%

pip-requirements.txt

@@ -14,4 +14,5 @@
 
 edk2-pytool-library==0.10.*
 edk2-pytool-extensions~=0.13.3
+edk2-basetools==0.1.2
 antlr4-python3-runtime==4.7.1