tyler.durden
/
audk
mirror of https://github.com/acidanthera/audk.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BaseTools: add support for BIOS build with binary cache 

Add three new options:
--hash enables hash-based caching during build process. when --hash is
enabled, build tool will base on the module hash value to do the
incremental build, without --hash, build tool will base on the
timestamp to do the incremental build. --hash option use md5 method to
get every hash value, DSC/FDF, tools_def.txt, build_rule.txt and build
command are calculated as global hash value, Package DEC and its
include header files are calculated as package hash value, Module
source files and its INF file are calculated as module hash value.
Library hash value will combine the global hash value and its dependent
package hash value. Driver hash value will combine the global hash
value, its dependent package hash value and its linked library hash
value.
When --hash and --binary-destination are specified, build tool will
copy generated binary files for each module into the directory specified
by binary-destination at the build phase. Binary-destination directory
caches all generated binary files.
When --hash and --binary-source are specified, build tool will try to
get the binary files from the binary source directory at the build
phase.If the cached binary has the same hash value, it will be directly
used. Otherwise, build tool will compile the source files and generate
the binary files.

Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yonghong Zhu <yonghong.zhu@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Yonghong Zhu 2017-09-19 14:31:09 +08:00
parent 3dcb53258d
commit 36d083ef00
3 changed files with 189 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
BaseTools/Source/Python/AutoGen/AutoGen.py
View File

@ -43,6 +43,7 @@ from Workspace.MetaFileCommentParser import UsageList
from Common.MultipleWorkspace import MultipleWorkspace as mws
import InfSectionParser
import datetime
import hashlib
## Regular expression for splitting Dependency Expression string into tokens
gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
@ -265,6 +266,10 @@ class WorkspaceAutoGen(AutoGen):
self.FvTargetList = Fvs
self.CapTargetList = Caps
self.AutoGenObjectList = []
self._BuildDir = None
self._FvDir = None
self._MakeFileDir = None
self._BuildCommand = None
# there's many relative directory operations, so ...
os.chdir(self.WorkspaceDir)
@ -644,6 +649,14 @@ class WorkspaceAutoGen(AutoGen):
Pa.CollectFixedAtBuildPcds()
self.AutoGenObjectList.append(Pa)
#
# Generate Package level hash value
#
GlobalData.gPackageHash[Arch] = {}
if GlobalData.gUseHashCache:
for Pkg in Pkgs:
self._GenPkgLevelHash(Pkg)
#
# Check PCDs token value conflict in each DEC file.
#
@ -657,11 +670,6 @@ class WorkspaceAutoGen(AutoGen):
# if self.FdfFile:
# self._CheckDuplicateInFV(Fdf)
self._BuildDir = None
self._FvDir = None
self._MakeFileDir = None
self._BuildCommand = None
#
# Create BuildOptions Macro & PCD metafile, also add the Active Platform and FDF file.
#
@ -677,6 +685,10 @@ class WorkspaceAutoGen(AutoGen):
if self.FdfFile:
content += 'Flash Image Definition: '
content += str(self.FdfFile)
content += os.linesep
if GlobalData.gBinCacheDest:
content += 'Cache of .efi location: '
content += str(GlobalData.gBinCacheDest)
SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False)
#
@ -706,6 +718,18 @@ class WorkspaceAutoGen(AutoGen):
SrcTimeStamp = os.stat(f)[8]
self._SrcTimeStamp = SrcTimeStamp
if GlobalData.gUseHashCache:
m = hashlib.md5()
for files in AllWorkSpaceMetaFiles:
if files.endswith('.dec'):
continue
f = open(files, 'r')
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), True)
GlobalData.gPlatformHash = m.hexdigest()
#
# Write metafile list to build directory
#
@ -719,6 +743,29 @@ class WorkspaceAutoGen(AutoGen):
print >> file, f
return True
def _GenPkgLevelHash(self, Pkg):
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
CreateDirectory(PkgDir)
HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')
m = hashlib.md5()
# Get .dec file's hash value
f = open(Pkg.MetaFile.Path, 'r')
Content = f.read()
f.close()
m.update(Content)
# Get include files hash value
if Pkg.Includes:
for inc in Pkg.Includes:
for Root, Dirs, Files in os.walk(str(inc)):
for File in Files:
File_Path = os.path.join(Root, File)
f = open(File_Path, 'r')
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(HashFile, m.hexdigest(), True)
if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]:
GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = m.hexdigest()
def _GetMetaFiles(self, Target, Toolchain, Arch):
AllWorkSpaceMetaFiles = set()
@ -956,7 +1003,8 @@ class WorkspaceAutoGen(AutoGen):
## Return the directory to store all intermediate and final files built
def _GetBuildDir(self):
return self.AutoGenObjectList[0].BuildDir
if self._BuildDir == None:
return self.AutoGenObjectList[0].BuildDir
## Return the build output directory platform specifies
def _GetOutputDir(self):
@ -3911,9 +3959,11 @@ class ModuleAutoGen(AutoGen):
AsBuiltInfDict['module_pi_specification_version'] += [self.Specification['PI_SPECIFICATION_VERSION']]
OutputDir = self.OutputDir.replace('\\', '/').strip('/')
self.OutputFile = []
for Item in self.CodaTargetList:
File = Item.Target.Path.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/')
if File not in self.OutputFile:
self.OutputFile.append(File)
if Item.Target.Ext.lower() == '.aml':
AsBuiltInfDict['binary_item'] += ['ASL|' + File]
elif Item.Target.Ext.lower() == '.acpi':
@ -3923,6 +3973,8 @@ class ModuleAutoGen(AutoGen):
else:
AsBuiltInfDict['binary_item'] += ['BIN|' + File]
if self.DepexGenerated:
if self.Name + '.depex' not in self.OutputFile:
self.OutputFile.append(self.Name + '.depex')
if self.ModuleType in ['PEIM']:
AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + '.depex']
if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER', 'DXE_SAL_DRIVER', 'UEFI_DRIVER']:
@ -3933,11 +3985,15 @@ class ModuleAutoGen(AutoGen):
Bin = self._GenOffsetBin()
if Bin:
AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin]
if Bin not in self.OutputFile:
self.OutputFile.append(Bin)
for Root, Dirs, Files in os.walk(OutputDir):
for File in Files:
if File.lower().endswith('.pdb'):
AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File]
if File not in self.OutputFile:
self.OutputFile.append(File)
HeaderComments = self.Module.HeaderComments
StartPos = 0
for Index in range(len(HeaderComments)):
@ -4118,7 +4174,48 @@ class ModuleAutoGen(AutoGen):
SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False)
self.IsAsBuiltInfCreated = True
if GlobalData.gBinCacheDest:
self.CopyModuleToCache()
def CopyModuleToCache(self):
FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, self.SourceDir, self.MetaFile.BaseName)
CreateDirectory (FileDir)
HashFile = path.join(self.BuildDir, self.Name + '.hash')
ModuleFile = path.join(self.OutputDir, self.Name + '.inf')
if os.path.exists(HashFile):
shutil.copy2(HashFile, FileDir)
if os.path.exists(ModuleFile):
shutil.copy2(ModuleFile, FileDir)
if self.OutputFile:
for File in self.OutputFile:
if not os.path.isabs(File):
File = os.path.join(self.OutputDir, File)
if os.path.exists(File):
shutil.copy2(File, FileDir)
def AttemptModuleCacheCopy(self):
if self.IsBinaryModule:
return False
FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, self.SourceDir, self.MetaFile.BaseName)
HashFile = path.join(FileDir, self.Name + '.hash')
if os.path.exists(HashFile):
f = open(HashFile, 'r')
CacheHash = f.read()
f.close()
if GlobalData.gModuleHash[self.Arch][self.Name]:
if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:
for root, dir, files in os.walk(FileDir):
for f in files:
if self.Name + '.hash' in f:
shutil.copy2(HashFile, self.BuildDir)
else:
File = path.join(root, f)
shutil.copy2(File, self.OutputDir)
if self.Name == "PcdPeim" or self.Name == "PcdDxe":
CreatePcdDatabaseCode(self, TemplateString(), TemplateString())
return True
return False
## Create makefile for the module and its dependent libraries
#
# @param CreateLibraryMakeFile Flag indicating if or not the makefiles of
@ -4246,8 +4343,54 @@ class ModuleAutoGen(AutoGen):
self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE)
return self._LibraryAutoGenList
def GenModuleHash(self):
if self.Arch not in GlobalData.gModuleHash:
GlobalData.gModuleHash[self.Arch] = {}
m = hashlib.md5()
# Add Platform level hash
m.update(GlobalData.gPlatformHash)
# Add Package level hash
if self.DependentPackageList:
for Pkg in self.DependentPackageList:
if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]:
m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName])
# Add Library hash
if self.LibraryAutoGenList:
for Lib in self.LibraryAutoGenList:
if Lib.Name not in GlobalData.gModuleHash[self.Arch]:
Lib.GenModuleHash()
m.update(GlobalData.gModuleHash[self.Arch][Lib.Name])
# Add Module self
f = open(str(self.MetaFile), 'r')
Content = f.read()
f.close()
m.update(Content)
# Add Module's source files
if self.SourceFileList:
for File in self.SourceFileList:
f = open(str(File), 'r')
Content = f.read()
f.close()
m.update(Content)
ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")
if self.Name not in GlobalData.gModuleHash[self.Arch]:
GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()
if GlobalData.gBinCacheSource:
CacheValid = self.AttemptModuleCacheCopy()
if CacheValid:
return False
return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True)
## Decide whether we can skip the ModuleAutoGen process
# If any source file is newer than the modeule than we cannot skip
def CanSkipbyHash(self):
if GlobalData.gUseHashCache:
return not self.GenModuleHash()
## Decide whether we can skip the ModuleAutoGen process
# If any source file is newer than the module than we cannot skip
#
def CanSkip(self):
if not os.path.exists(self.GetTimeStampPath()):

7
BaseTools/Source/Python/Common/GlobalData.py
View File

@ -87,3 +87,10 @@ MixedPcd = {}
# Pcd name for the Pcd which used in the Conditional directives
gConditionalPcds = []
gUseHashCache = None
gBinCacheDest = None
gBinCacheSource = None
gPlatformHash = None
gPackageHash = {}
gModuleHash = {}

30
BaseTools/Source/Python/build/build.py
View File

@ -771,6 +771,30 @@ class Build():
GlobalData.BuildOptionPcd = BuildOptions.OptionPcd
#Set global flag for build mode
GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
GlobalData.gUseHashCache = BuildOptions.UseHashCache
GlobalData.gBinCacheDest = BuildOptions.BinCacheDest
GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
if GlobalData.gBinCacheSource:
BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
if not os.path.isabs(BinCacheSource):
BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
GlobalData.gBinCacheSource = BinCacheSource
if GlobalData.gBinCacheDest:
BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
if not os.path.isabs(BinCacheDest):
BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
GlobalData.gBinCacheDest = BinCacheDest
if self.ConfDirectory:
# Get alternate Conf location, if it is absolute, then just use the absolute directory name
@ -1952,6 +1976,9 @@ class Build():
if Ma == None:
continue
if Ma.CanSkipbyHash():
continue
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
@ -2274,6 +2301,9 @@ def MyOptionParser():
Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")
Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: \"PcdName=Value\" ")
Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")
Parser.add_option("--hash", action="store_true", dest="UseHashCache", default=False, help="Enable hash-based caching during build process.")
Parser.add_option("--binary-destination", action="store", type="string", dest="BinCacheDest", help="Generate a cache of binary files in the specified directory.")
Parser.add_option("--binary-source", action="store", type="string", dest="BinCacheSource", help="Consume a cache of binary files from the specified directory.")
(Opt, Args) = Parser.parse_args()
return (Opt, Args)