tyler.durden
/
audk
mirror of https://github.com/acidanthera/audk.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BaseTools: Refactor hash tracking after checking for Sources section 

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804

After adding a check to see if [Sources] section lists all the header
type files of a module, track module and library hashes for --hash
feature. If above check is not in compilance for a library or module,
force hash invalidation on that library or module.

Signed-off-by: Christian Rodriguez <christian.rodriguez@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Rodriguez, Christian 2019-05-30 00:26:49 +08:00 committed by Feng, Bob C
parent 1fa6699e6c
commit 48b0bf6476
4 changed files with 53 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BaseTools/Source/Python/AutoGen/AutoGen.py
View File

@ -3995,7 +3995,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList: for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateMakeFile() LibraryAutoGen.CreateMakeFile()
if self.CanSkip(): # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
if not GlobalData.gUseHashCache and self.CanSkip():
return return
if len(self.CustomMakefile) == 0: if len(self.CustomMakefile) == 0:
@ -4038,7 +4039,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList: for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateCodeFile() LibraryAutoGen.CreateCodeFile()
if self.CanSkip(): # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
if not GlobalData.gUseHashCache and self.CanSkip():
return return
AutoGenList = [] AutoGenList = []

6
BaseTools/Source/Python/AutoGen/GenMake.py
View File

@ -935,10 +935,16 @@ cleanlib:
continue continue
headerFileDependencySet.add(aFileName) headerFileDependencySet.add(aFileName)
# Ensure that gModuleBuildTracking has been initialized per architecture
if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking:
GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] = dict()
# Check if a module dependency header file is missing from the module's MetaFile # Check if a module dependency header file is missing from the module's MetaFile
for aFile in headerFileDependencySet: for aFile in headerFileDependencySet:
if aFile in headerFilesInMetaFileSet: if aFile in headerFilesInMetaFileSet:
continue continue
if GlobalData.gUseHashCache:
GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGenObject] = 'FAIL_METAFILE'
EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!", EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!",
ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path
) )

3
BaseTools/Source/Python/Common/GlobalData.py
View File

@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False
gSikpAutoGenCache = set() gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure # Dictionary for tracking Module build status as success or failure
# False -> Fail : True -> Success # Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
gModuleBuildTracking = dict() gModuleBuildTracking = dict()
# Dictionary of booleans that dictate whether a module or # Dictionary of booleans that dictate whether a module or

53
BaseTools/Source/Python/build/build.py
View File

@ -625,8 +625,16 @@ class BuildTask:
BuildTask._ErrorFlag.set() BuildTask._ErrorFlag.set()
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \
(threading.currentThread().getName(), Command, WorkingDir) (threading.currentThread().getName(), Command, WorkingDir)
if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet():
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'
# If Module or Lib is being tracked, it did not fail header check test, and built successfully
if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking and
self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] != 'FAIL_METAFILE' and
not BuildTask._ErrorFlag.isSet()
):
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] = 'SUCCESS'
# indicate there's a thread is available for another build task # indicate there's a thread is available for another build task
BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueueLock.acquire()
BuildTask._RunningQueue.pop(self.BuildItem) BuildTask._RunningQueue.pop(self.BuildItem)
@ -1154,25 +1162,28 @@ class Build():
# #
# #
def invalidateHash(self): def invalidateHash(self):
# GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash # Only for hashing feature
for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): if not GlobalData.gUseHashCache:
# False == FAIL : True == Success return
# Skip invalidating for Successful module builds
if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True: # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash
for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys():
for moduleAutoGenObj in GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys():
# Skip invalidating for Successful Module/Lib builds
if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGenObj] == 'SUCCESS':
continue continue
# The module failed to build or failed to start building, from this point on # The module failed to build, failed to start building, or failed the header check test from this point on
# Remove .hash from build # Remove .hash from build
if GlobalData.gUseHashCache: ModuleHashFile = os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")
ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")
if os.path.exists(ModuleHashFile): if os.path.exists(ModuleHashFile):
os.remove(ModuleHashFile) os.remove(ModuleHashFile)
# Remove .hash file from cache # Remove .hash file from cache
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
FileDir = path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) FileDir = os.path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)
HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash') HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + '.hash')
if os.path.exists(HashFile): if os.path.exists(HashFile):
os.remove(HashFile) os.remove(HashFile)
@ -1825,9 +1836,11 @@ class Build():
if self.Target == "genmake": if self.Target == "genmake":
return True return True
self.BuildModules.append(Ma) self.BuildModules.append(Ma)
# Initialize all modules in tracking to False (FAIL) # Initialize all modules in tracking to 'FAIL'
if Ma not in GlobalData.gModuleBuildTracking: if Ma.Arch not in GlobalData.gModuleBuildTracking:
GlobalData.gModuleBuildTracking[Ma] = False GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
self.AutoGenTime += int(round((time.time() - AutoGenStart))) self.AutoGenTime += int(round((time.time() - AutoGenStart)))
MakeStart = time.time() MakeStart = time.time()
for Ma in self.BuildModules: for Ma in self.BuildModules:
@ -1911,6 +1924,7 @@ class Build():
# Save MAP buffer into MAP file. # Save MAP buffer into MAP file.
# #
self._SaveMapFile (MapBuffer, Wa) self._SaveMapFile (MapBuffer, Wa)
self.invalidateHash()
def _GenFfsCmd(self,ArchList): def _GenFfsCmd(self,ArchList):
# convert dictionary of Cmd:(Inf,Arch) # convert dictionary of Cmd:(Inf,Arch)
@ -2009,9 +2023,11 @@ class Build():
if self.Target == "genmake": if self.Target == "genmake":
continue continue
self.BuildModules.append(Ma) self.BuildModules.append(Ma)
# Initialize all modules in tracking to False (FAIL) # Initialize all modules in tracking to 'FAIL'
if Ma not in GlobalData.gModuleBuildTracking: if Ma.Arch not in GlobalData.gModuleBuildTracking:
GlobalData.gModuleBuildTracking[Ma] = False GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
self.Progress.Stop("done!") self.Progress.Stop("done!")
self.AutoGenTime += int(round((time.time() - AutoGenStart))) self.AutoGenTime += int(round((time.time() - AutoGenStart)))
MakeStart = time.time() MakeStart = time.time()
@ -2099,6 +2115,7 @@ class Build():
# Save MAP buffer into MAP file. # Save MAP buffer into MAP file.
# #
self._SaveMapFile(MapBuffer, Wa) self._SaveMapFile(MapBuffer, Wa)
self.invalidateHash()
## Generate GuidedSectionTools.txt in the FV directories. ## Generate GuidedSectionTools.txt in the FV directories.
# #