tyler.durden/audk
1
0
Fork 0
mirror of https://github.com/acidanthera/audk.git synced 2025-04-08 17:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity
audk/BaseTools/Edk2ToolsBuild.py
kuqin12 3c8016b302 BaseTools: Support custom library build for base tools on Linux ARM 
This change added the build script to cross compile the base tool
binaries for Linux ARM/AARCH64 systems.

The needed libuuid system library is pulled from source file and rebuilt
to support the corresponding library dependencies. Individual tools'
makefiles are also updated to link the cross compiled library as well.

The EDK2 base tool build script was also updated to support such change.

This was tested functional on Linux ARM host system.

Cc: Rebecca Cran <rebecca@bsdio.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Signed-off-by: Kun Qin <kun.qin@microsoft.com>
2024-12-20 22:57:04 +00:00

378 lines
17 KiB
Python
Raw Blame History

# @file Edk2ToolsBuild.py
# Invocable class that builds the basetool c files.
#
# Supports VS2019, VS2022, and GCC5
##
# Copyright (c) Microsoft Corporation
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import os
import sys
import logging
import argparse
import multiprocessing
import shutil
from edk2toolext import edk2_logging
from edk2toolext.environment import self_describing_environment, shell_environment
from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
from edk2toollib.utility_functions import RunCmd, GetHostInfo
from edk2toollib.windows.locate_tools import QueryVcVariables
class Edk2ToolsBuild(BaseAbstractInvocable):
def ParseCommandLineOptions(self):
''' parse arguments '''
ParserObj = argparse.ArgumentParser()
ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
help="Set the toolchain used to compile the build tools")
ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
"behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
args = ParserObj.parse_args()
self.tool_chain_tag = args.tct
self.target_arch = args.arch
def GetWorkspaceRoot(self):
''' Return the workspace root for initializing the SDE '''
# this is the bastools dir...not the traditional EDK2 workspace root
return os.path.dirname(os.path.abspath(__file__))
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
# Adding scope for cross compilers when building for ARM/AARCH64
scopes = ('global',)
if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
if self.target_arch is None:
return scopes
if "AARCH64" in self.target_arch:
scopes += ("gcc_aarch64_linux",)
if "ARM" in self.target_arch:
scopes += ("gcc_arm_linux",)
return scopes
def GetLoggingLevel(self, loggerType):
''' Get the logging level for a given type (return Logging.Level)
base == lowest logging level supported
con == Screen logging
txt == plain text file logging
md == markdown file logging
'''
if(loggerType == "con"):
return logging.ERROR
else:
return logging.DEBUG
def GetLoggingFolderRelativeToRoot(self):
''' Return a path to folder for log files '''
return "BaseToolsBuild"
def GetVerifyCheckRequired(self):
''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
return True
def GetLoggingFileName(self, loggerType):
''' Get the logging file name for the type.
Return None if the logger shouldn't be created
base == lowest logging level supported
con == Screen logging
txt == plain text file logging
md == markdown file logging
'''
return "BASETOOLS_BUILD"
def WritePathEnvFile(self, OutputDir):
''' Write a PyTool path env file for future PyTool based edk2 builds'''
content = '''##
# Set shell variable EDK_TOOLS_BIN to this folder
#
# Autogenerated by Edk2ToolsBuild.py
#
# Copyright (c), Microsoft Corporation
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"id": "You-Built-BaseTools",
"scope": "edk2-build",
"flags": ["set_shell_var", "set_path"],
"var_name": "EDK_TOOLS_BIN"
}
'''
with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
f.write(content)
def Go(self):
logging.info("Running Python version: " + str(sys.version_info))
(build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
# # Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
if self.tool_chain_tag.lower().startswith("vs"):
if self.target_arch is None:
# Put a default as IA32
self.target_arch = "IA32"
if self.target_arch == "IA32":
VcToolChainArch = "x86"
TargetInfoArch = "x86"
OutputDir = "Win32"
elif self.target_arch == "ARM":
VcToolChainArch = "x86_arm"
TargetInfoArch = "ARM"
OutputDir = "Win32"
elif self.target_arch == "X64":
VcToolChainArch = "amd64"
TargetInfoArch = "x86"
OutputDir = "Win64"
elif self.target_arch == "AARCH64":
VcToolChainArch = "amd64_arm64"
TargetInfoArch = "ARM"
OutputDir = "Win64"
else:
raise NotImplementedError()
self.OutputDir = os.path.join(
shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)
# compiled tools need to be added to path because antlr is referenced
HostInfo = GetHostInfo()
if TargetInfoArch == HostInfo.arch:
# not cross compiling
shell_env.insert_path(self.OutputDir)
else:
# cross compiling:
# as the VfrCompile tool is needed in the build process, we need
# to build one for the host system, then add the path to the
# tools to the PATH environment variable
shell_environment.CheckpointBuildVars()
if HostInfo.arch == "x86" and HostInfo.bit == "64":
host_arch = "X64"
host_toolchain_arch = "amd64"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
elif HostInfo.arch == "x86" and HostInfo.bit == "32":
host_arch = "IA32"
host_toolchain_arch = "x86"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
host_arch = "AARCH64"
host_toolchain_arch = "amd64_arm64"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
host_arch = "ARM"
host_toolchain_arch = "x86_arm"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
else:
raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))
interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
interesting_keys.extend(
["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
interesting_keys.extend(
["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
vc_vars = QueryVcVariables(
interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
for key in vc_vars.keys():
logging.debug(f"Var - {key} = {vc_vars[key]}")
if key.lower() == 'path':
shell_env.set_path(vc_vars[key])
else:
shell_env.set_shell_var(key, vc_vars[key])
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
shell_env.set_shell_var('HOST_ARCH', host_arch)
shell_env.insert_path(TempOutputDir)
# All set, build the tools for the host system.
ret = RunCmd('nmake.exe', None,
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
if ret != 0:
raise Exception("Failed to build base tools for host system.")
# Copy the output to a temp directory
TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
if not os.path.exists(TempFolder):
os.makedirs(TempFolder)
for file in os.listdir(TempOutputDir):
shutil.copy(os.path.join(TempOutputDir, file), TempFolder)
# Clean up the build output
ret = RunCmd('nmake.exe', 'cleanall',
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
# Remove the entire TempOutputDir
shutil.rmtree(TempOutputDir)
shell_environment.RevertBuildVars()
shell_env.insert_path(TempFolder)
# # Update environment with required VC vars.
interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
interesting_keys.extend(
["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
interesting_keys.extend(
["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
vc_vars = QueryVcVariables(
interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
for key in vc_vars.keys():
logging.debug(f"Var - {key} = {vc_vars[key]}")
if key.lower() == 'path':
shell_env.set_path(vc_vars[key])
else:
shell_env.set_shell_var(key, vc_vars[key])
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
shell_env.set_shell_var('HOST_ARCH', self.target_arch)
# Actually build the tools.
output_stream = edk2_logging.create_output_stream()
ret = RunCmd('nmake.exe', None,
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
edk2_logging.remove_output_stream(output_stream)
problems = edk2_logging.scan_compiler_output(output_stream)
for level, problem in problems:
logging.log(level, problem)
if ret != 0:
raise Exception("Failed to build.")
self.WritePathEnvFile(self.OutputDir)
return ret
elif self.tool_chain_tag.lower().startswith("gcc"):
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
HostInfo = GetHostInfo()
prefix = None
TargetInfoArch = None
if self.target_arch is not None:
shell_env.set_shell_var('HOST_ARCH', self.target_arch)
if "AARCH64" in self.target_arch:
prefix = shell_env.get_shell_var("GCC5_AARCH64_PREFIX")
if prefix == None:
# now check for install dir. If set then set the Prefix
install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")
# make GCC5_AARCH64_PREFIX to align with tools_def.txt
prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
TargetInfoArch = "ARM"
elif "ARM" in self.target_arch:
prefix = shell_env.get_shell_var("GCC5_ARM_PREFIX")
if prefix == None:
# now check for install dir. If set then set the Prefix
install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
# make GCC5_ARM_PREFIX to align with tools_def.txt
prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
TargetInfoArch = "ARM"
else:
TargetInfoArch = "x86"
else:
self.target_arch = HostInfo.arch
TargetInfoArch = HostInfo.arch
# Otherwise, the built binary arch will be consistent with the host system
# Added logic to support cross compilation scenarios
if TargetInfoArch != HostInfo.arch:
# this is defaulting to the version that comes with Ubuntu 20.04
ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
pack_name = f"util-linux-{ver}"
unzip_dir = os.path.join(work_dir, pack_name)
if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
if os.path.isdir(unzip_dir):
shutil.rmtree(unzip_dir)
# cross compiling, need to rebuild libuuid for the target
ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
if ret != 0:
raise Exception(f"Failed to download libuuid version {ver} - {ret}")
ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
if ret != 0:
raise Exception(f"Failed to untar the downloaded file {ret}")
# configure the source to use the cross compiler
pack_name = f"util-linux-{ver}"
if "AARCH64" in self.target_arch:
ret = RunCmd("sh", f"./configure --host=aarch64-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
elif "ARM" in self.target_arch:
ret = RunCmd("sh", f"./configure --host=arm-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
if ret != 0:
raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")
ret = RunCmd("make", "", workingdir=unzip_dir)
if ret != 0:
raise Exception(f"Failed to build the libuuid with our gcc {ret}")
shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))
ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
if ret != 0:
raise Exception("Failed to build.")
cpu_count = self.GetCpuThreads()
output_stream = edk2_logging.create_output_stream()
ret = RunCmd("make", f"-C . -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
edk2_logging.remove_output_stream(output_stream)
problems = edk2_logging.scan_compiler_output(output_stream)
for level, problem in problems:
logging.log(level, problem)
if ret != 0:
raise Exception("Failed to build.")
self.OutputDir = os.path.join(
shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")
self.WritePathEnvFile(self.OutputDir)
return ret
logging.critical("Tool Chain not supported")
return -1
def GetCpuThreads(self) -> int:
''' Function to return number of cpus. If error return 1'''
cpus = 1
try:
cpus = multiprocessing.cpu_count()
except:
# from the internet there are cases where cpu_count is not implemented.
# will handle error by just doing single proc build
pass
return cpus
def main():
Edk2ToolsBuild().Invoke()
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink