mirror of https://github.com/acidanthera/audk.git
BaseTools: Enable Multiple Process AutoGen
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 Assign the Module AutoGen tasks into multiple sub process. Cc: Liming Gao <liming.gao@intel.com> Signed-off-by: Bob Feng <bob.c.feng@intel.com> Acked-by: Laszlo Ersek <lersek@redhat.com> Tested-by: Laszlo Ersek <lersek@redhat.com> Acked-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Feng, Bob C
parent
e8449e1d8e
commit
673d09a2dd
|
|
@ -0,0 +1,191 @@
|
|||
## @file
|
||||
# Create makefile for MS nmake and GNU make
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
|
||||
# SPDX-License-Identifier: BSD-2-Clause-Patent
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
import multiprocessing as mp
|
||||
import threading
|
||||
from Common.Misc import PathClass
|
||||
from AutoGen.ModuleAutoGen import ModuleAutoGen
|
||||
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
|
||||
import Common.GlobalData as GlobalData
|
||||
import Common.EdkLogger as EdkLogger
|
||||
import os
|
||||
from Common.MultipleWorkspace import MultipleWorkspace as mws
|
||||
from AutoGen.AutoGen import AutoGen
|
||||
from Workspace.WorkspaceDatabase import BuildDB
|
||||
from queue import Empty
|
||||
import traceback
|
||||
import sys
|
||||
from AutoGen.DataPipe import MemoryDataPipe
|
||||
def clearQ(q):
|
||||
try:
|
||||
while True:
|
||||
q.get_nowait()
|
||||
except Empty:
|
||||
pass
|
||||
class AutoGenManager(threading.Thread):
|
||||
def __init__(self,autogen_workers, feedback_q,error_event):
|
||||
super(AutoGenManager,self).__init__()
|
||||
self.autogen_workers = autogen_workers
|
||||
self.feedback_q = feedback_q
|
||||
self.terminate = False
|
||||
self.Status = True
|
||||
self.error_event = error_event
|
||||
def run(self):
|
||||
try:
|
||||
fin_num = 0
|
||||
while True:
|
||||
badnews = self.feedback_q.get()
|
||||
if badnews is None:
|
||||
break
|
||||
if badnews == "Done":
|
||||
fin_num += 1
|
||||
else:
|
||||
self.Status = False
|
||||
self.TerminateWorkers()
|
||||
if fin_num == len(self.autogen_workers):
|
||||
self.clearQueue()
|
||||
for w in self.autogen_workers:
|
||||
w.join()
|
||||
break
|
||||
except Exception:
|
||||
return
|
||||
|
||||
def clearQueue(self):
|
||||
taskq = self.autogen_workers[0].module_queue
|
||||
clearQ(taskq)
|
||||
clearQ(self.feedback_q)
|
||||
|
||||
def TerminateWorkers(self):
|
||||
self.error_event.set()
|
||||
def kill(self):
|
||||
self.feedback_q.put(None)
|
||||
class AutoGenWorkerInProcess(mp.Process):
|
||||
def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event):
|
||||
mp.Process.__init__(self)
|
||||
self.module_queue = module_queue
|
||||
self.data_pipe_file_path =data_pipe_file_path
|
||||
self.data_pipe = None
|
||||
self.feedback_q = feedback_q
|
||||
self.PlatformMetaFileSet = {}
|
||||
self.file_lock = file_lock
|
||||
self.error_event = error_event
|
||||
def GetPlatformMetaFile(self,filepath,root):
|
||||
try:
|
||||
return self.PlatformMetaFileSet[(filepath,root)]
|
||||
except:
|
||||
self.PlatformMetaFileSet[(filepath,root)] = filepath
|
||||
return self.PlatformMetaFileSet[(filepath,root)]
|
||||
def run(self):
|
||||
try:
|
||||
taskname = "Init"
|
||||
with self.file_lock:
|
||||
if not os.path.exists(self.data_pipe_file_path):
|
||||
self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
|
||||
self.data_pipe = MemoryDataPipe()
|
||||
self.data_pipe.load(self.data_pipe_file_path)
|
||||
EdkLogger.Initialize()
|
||||
loglevel = self.data_pipe.Get("LogLevel")
|
||||
if not loglevel:
|
||||
loglevel = EdkLogger.INFO
|
||||
EdkLogger.SetLevel(loglevel)
|
||||
logfile = self.data_pipe.Get("LogFile")
|
||||
if logfile:
|
||||
EdkLogger.SetLogFile(logfile)
|
||||
target = self.data_pipe.Get("P_Info").get("Target")
|
||||
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
|
||||
archlist = self.data_pipe.Get("P_Info").get("ArchList")
|
||||
|
||||
active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
|
||||
workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
|
||||
PackagesPath = os.getenv("PACKAGES_PATH")
|
||||
mws.setWs(workspacedir, PackagesPath)
|
||||
self.Wa = WorkSpaceInfo(
|
||||
workspacedir,active_p,target,toolchain,archlist
|
||||
)
|
||||
self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
|
||||
GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
|
||||
GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
|
||||
os.environ._data = self.data_pipe.Get("Env_Var")
|
||||
GlobalData.gWorkspace = workspacedir
|
||||
GlobalData.gDisableIncludePathCheck = False
|
||||
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
|
||||
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
|
||||
pcd_from_build_option = []
|
||||
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
|
||||
pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
|
||||
if pcd_tuple[2].strip():
|
||||
pcd_id = ".".join((pcd_id,pcd_tuple[2]))
|
||||
pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
|
||||
GlobalData.BuildOptionPcd = pcd_from_build_option
|
||||
module_count = 0
|
||||
FfsCmd = self.data_pipe.Get("FfsCommand")
|
||||
if FfsCmd is None:
|
||||
FfsCmd = {}
|
||||
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
|
||||
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
|
||||
libConstPcd = self.data_pipe.Get("LibConstPcd")
|
||||
Refes = self.data_pipe.Get("REFS")
|
||||
while True:
|
||||
if self.module_queue.empty():
|
||||
break
|
||||
if self.error_event.is_set():
|
||||
break
|
||||
module_count += 1
|
||||
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
|
||||
modulefullpath = os.path.join(module_root,module_file)
|
||||
taskname = " : ".join((modulefullpath,module_arch))
|
||||
module_metafile = PathClass(module_file,module_root)
|
||||
if module_path:
|
||||
module_metafile.Path = module_path
|
||||
if module_basename:
|
||||
module_metafile.BaseName = module_basename
|
||||
if module_originalpath:
|
||||
module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
|
||||
arch = module_arch
|
||||
target = self.data_pipe.Get("P_Info").get("Target")
|
||||
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
|
||||
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
|
||||
Ma.IsLibrary = IsLib
|
||||
if IsLib:
|
||||
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
|
||||
Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
|
||||
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
|
||||
Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
|
||||
Ma.CreateCodeFile(False)
|
||||
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
|
||||
except Empty:
|
||||
pass
|
||||
except:
|
||||
traceback.print_exc(file=sys.stdout)
|
||||
self.feedback_q.put(taskname)
|
||||
finally:
|
||||
self.feedback_q.put("Done")
|
||||
def printStatus(self):
|
||||
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
|
||||
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
|
||||
groupobj = {}
|
||||
for buildobj in BuildDB.BuildObject.GetCache().values():
|
||||
if str(buildobj).lower().endswith("dec"):
|
||||
try:
|
||||
groupobj['dec'].append(str(buildobj))
|
||||
except:
|
||||
groupobj['dec'] = [str(buildobj)]
|
||||
if str(buildobj).lower().endswith("dsc"):
|
||||
try:
|
||||
groupobj['dsc'].append(str(buildobj))
|
||||
except:
|
||||
groupobj['dsc'] = [str(buildobj)]
|
||||
|
||||
if str(buildobj).lower().endswith("inf"):
|
||||
try:
|
||||
groupobj['inf'].append(str(buildobj))
|
||||
except:
|
||||
groupobj['inf'] = [str(buildobj)]
|
||||
|
||||
print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
|
||||
print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
|
||||
print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))
|
||||
|
|
@ -11,6 +11,7 @@ import Common.GlobalData as GlobalData
|
|||
import os
|
||||
import pickle
|
||||
from pickle import HIGHEST_PROTOCOL
|
||||
from Common import EdkLogger
|
||||
|
||||
class PCD_DATA():
|
||||
def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue,
|
||||
|
|
@ -34,6 +35,7 @@ class DataPipe(object):
|
|||
def __init__(self, BuildDir=None):
|
||||
self.data_container = {}
|
||||
self.BuildDir = BuildDir
|
||||
self.dump_file = ""
|
||||
|
||||
class MemoryDataPipe(DataPipe):
|
||||
|
||||
|
|
@ -41,6 +43,7 @@ class MemoryDataPipe(DataPipe):
|
|||
return self.data_container.get(key)
|
||||
|
||||
def dump(self,file_path):
|
||||
self.dump_file = file_path
|
||||
with open(file_path,'wb') as fd:
|
||||
pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
|
|
@ -71,7 +74,7 @@ class MemoryDataPipe(DataPipe):
|
|||
for m in PlatformInfo.Platform.Modules:
|
||||
m_pcds = PlatformInfo.Platform.Modules[m].Pcds
|
||||
if m_pcds:
|
||||
ModulePcds[(m.File,m.Root)] = [PCD_DATA(
|
||||
ModulePcds[(m.File,m.Root,m.Arch)] = [PCD_DATA(
|
||||
pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type,
|
||||
pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue,
|
||||
pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges,
|
||||
|
|
@ -83,11 +86,18 @@ class MemoryDataPipe(DataPipe):
|
|||
|
||||
#Module's Library Instance
|
||||
ModuleLibs = {}
|
||||
libModules = {}
|
||||
for m in PlatformInfo.Platform.Modules:
|
||||
module_obj = BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain]
|
||||
Libs = GetModuleLibInstances(module_obj, PlatformInfo.Platform, BuildDB.BuildObject, PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain)
|
||||
ModuleLibs[(m.File,m.Root,module_obj.Arch)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch) for l in Libs]
|
||||
for lib in Libs:
|
||||
try:
|
||||
libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)].append((m.File,m.Root,module_obj.Arch,m.Path))
|
||||
except:
|
||||
libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)] = [(m.File,m.Root,module_obj.Arch,m.Path)]
|
||||
ModuleLibs[(m.File,m.Root,module_obj.Arch,m.Path)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch,l.MetaFile.Path) for l in Libs]
|
||||
self.DataContainer = {"DEPS":ModuleLibs}
|
||||
self.DataContainer = {"REFS":libModules}
|
||||
|
||||
#Platform BuildOptions
|
||||
|
||||
|
|
@ -143,5 +153,8 @@ class MemoryDataPipe(DataPipe):
|
|||
|
||||
self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict}
|
||||
|
||||
self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath}
|
||||
self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False}
|
||||
|
||||
self.DataContainer = {"LogLevel": EdkLogger.GetLevel()}
|
||||
self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""}
|
||||
|
|
|
|||
|
|
@ -1686,6 +1686,7 @@ class ModuleAutoGen(AutoGen):
|
|||
if not self.IsLibrary and CreateLibraryMakeFile:
|
||||
for LibraryAutoGen in self.LibraryAutoGenList:
|
||||
LibraryAutoGen.CreateMakeFile()
|
||||
|
||||
# Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
|
||||
if not GlobalData.gUseHashCache and self.CanSkip():
|
||||
return
|
||||
|
|
@ -1729,7 +1730,6 @@ class ModuleAutoGen(AutoGen):
|
|||
if not self.IsLibrary and CreateLibraryCodeFile:
|
||||
for LibraryAutoGen in self.LibraryAutoGenList:
|
||||
LibraryAutoGen.CreateCodeFile()
|
||||
|
||||
# Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping
|
||||
if not GlobalData.gUseHashCache and self.CanSkip():
|
||||
return
|
||||
|
|
|
|||
|
|
@ -133,6 +133,12 @@ class PlatformAutoGen(AutoGen):
|
|||
self.DataPipe.FillData(self)
|
||||
|
||||
return True
|
||||
def FillData_LibConstPcd(self):
|
||||
libConstPcd = {}
|
||||
for LibAuto in self.LibraryAutoGenList:
|
||||
if LibAuto.ConstPcd:
|
||||
libConstPcd[(LibAuto.MetaFile.File,LibAuto.MetaFile.Root,LibAuto.Arch,LibAuto.MetaFile.Path)] = LibAuto.ConstPcd
|
||||
self.DataPipe.DataContainer = {"LibConstPcd":libConstPcd}
|
||||
## hash() operator of PlatformAutoGen
|
||||
#
|
||||
# The platform file path and arch string will be used to represent
|
||||
|
|
@ -162,7 +168,7 @@ class PlatformAutoGen(AutoGen):
|
|||
return
|
||||
|
||||
for Ma in self.ModuleAutoGenList:
|
||||
Ma.CreateCodeFile(True)
|
||||
Ma.CreateCodeFile(CreateModuleCodeFile)
|
||||
|
||||
## Generate Fds Command
|
||||
@cached_property
|
||||
|
|
@ -179,9 +185,9 @@ class PlatformAutoGen(AutoGen):
|
|||
for Ma in self._MaList:
|
||||
key = (Ma.MetaFile.File, self.Arch)
|
||||
if key in FfsCommand:
|
||||
Ma.CreateMakeFile(True, FfsCommand[key])
|
||||
Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
|
||||
else:
|
||||
Ma.CreateMakeFile(True)
|
||||
Ma.CreateMakeFile(CreateModuleMakeFile)
|
||||
|
||||
# no need to create makefile for the platform more than once
|
||||
if self.IsMakeFileCreated:
|
||||
|
|
@ -1086,10 +1092,10 @@ class PlatformAutoGen(AutoGen):
|
|||
Libs = GetModuleLibInstances(module_obj, self.Platform, self.BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain)
|
||||
else:
|
||||
Libs = []
|
||||
ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arch,True) for l in Libs]))
|
||||
ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.MetaFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l in Libs]))
|
||||
if WithoutPcd and module_obj.PcdIsDriver:
|
||||
continue
|
||||
ModuleLibs.add((m.File,m.Root,module_obj.Arch,False))
|
||||
ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath,module_obj.Arch,bool(module_obj.LibraryClass)))
|
||||
|
||||
return ModuleLibs
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,8 @@ class WorkspaceAutoGen(AutoGen):
|
|||
self.ProcessMixedPcd()
|
||||
self.VerifyPcdsFromFDF()
|
||||
self.CollectAllPcds()
|
||||
for Pa in self.AutoGenObjectList:
|
||||
Pa.FillData_LibConstPcd()
|
||||
self.GeneratePkgLevelHash()
|
||||
#
|
||||
# Check PCDs token value conflict in each DEC file.
|
||||
|
|
@ -881,7 +883,7 @@ class WorkspaceAutoGen(AutoGen):
|
|||
if not CreateDepsMakeFile:
|
||||
return
|
||||
for Pa in self.AutoGenObjectList:
|
||||
Pa.CreateMakeFile(True)
|
||||
Pa.CreateMakeFile(CreateDepsMakeFile)
|
||||
|
||||
## Create autogen code for platform and modules
|
||||
#
|
||||
|
|
@ -895,7 +897,7 @@ class WorkspaceAutoGen(AutoGen):
|
|||
if not CreateDepsCodeFile:
|
||||
return
|
||||
for Pa in self.AutoGenObjectList:
|
||||
Pa.CreateCodeFile(True)
|
||||
Pa.CreateCodeFile(CreateDepsCodeFile)
|
||||
|
||||
## Create AsBuilt INF file the platform
|
||||
#
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from optparse import OptionParser
|
|||
from AutoGen.PlatformAutoGen import PlatformAutoGen
|
||||
from AutoGen.ModuleAutoGen import ModuleAutoGen
|
||||
from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
|
||||
from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager
|
||||
from AutoGen import GenMake
|
||||
from Common import Misc as Utils
|
||||
|
||||
|
|
@ -50,7 +51,7 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
|
|||
|
||||
import Common.GlobalData as GlobalData
|
||||
from GenFds.GenFds import GenFds, GenFdsApi
|
||||
|
||||
import multiprocessing as mp
|
||||
|
||||
# Version and Copyright
|
||||
VersionNumber = "0.60" + ' ' + gBUILD_VERSION
|
||||
|
|
@ -343,9 +344,9 @@ class ModuleMakeUnit(BuildUnit):
|
|||
# @param Obj The ModuleAutoGen object the build is working on
|
||||
# @param Target The build target name, one of gSupportedTarget
|
||||
#
|
||||
def __init__(self, Obj, Target):
|
||||
Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]
|
||||
BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
|
||||
def __init__(self, Obj, BuildCommand,Target):
|
||||
Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
|
||||
BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
|
||||
if Target in [None, "", "all"]:
|
||||
self.Target = "tbuild"
|
||||
|
||||
|
|
@ -364,10 +365,10 @@ class PlatformMakeUnit(BuildUnit):
|
|||
# @param Obj The PlatformAutoGen object the build is working on
|
||||
# @param Target The build target name, one of gSupportedTarget
|
||||
#
|
||||
def __init__(self, Obj, Target):
|
||||
Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]
|
||||
Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])
|
||||
BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
|
||||
def __init__(self, Obj, BuildCommand, Target):
|
||||
Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
|
||||
Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
|
||||
BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
|
||||
|
||||
## The class representing the task of a module build or platform build
|
||||
#
|
||||
|
|
@ -824,8 +825,31 @@ class Build():
|
|||
if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
|
||||
self.InitBuild()
|
||||
|
||||
self.AutoGenMgr = None
|
||||
EdkLogger.info("")
|
||||
os.chdir(self.WorkspaceDir)
|
||||
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList):
|
||||
try:
|
||||
if SkipAutoGen:
|
||||
return True,0
|
||||
feedback_q = mp.Queue()
|
||||
file_lock = mp.Lock()
|
||||
error_event = mp.Event()
|
||||
auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)]
|
||||
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
|
||||
self.AutoGenMgr.start()
|
||||
for w in auto_workers:
|
||||
w.start()
|
||||
if PcdMaList is not None:
|
||||
for PcdMa in PcdMaList:
|
||||
PcdMa.CreateCodeFile(False)
|
||||
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
|
||||
|
||||
self.AutoGenMgr.join()
|
||||
rt = self.AutoGenMgr.Status
|
||||
return rt, 0
|
||||
except Exception as e:
|
||||
return False,e.errcode
|
||||
|
||||
## Load configuration
|
||||
#
|
||||
|
|
@ -1190,30 +1214,34 @@ class Build():
|
|||
# @param CreateDepModuleMakeFile Flag used to indicate creating makefile
|
||||
# for dependent modules/Libraries
|
||||
#
|
||||
def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}):
|
||||
def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
|
||||
if AutoGenObject is None:
|
||||
return False
|
||||
|
||||
if FfsCommand is None:
|
||||
FfsCommand = {}
|
||||
# skip file generation for cleanxxx targets, run and fds target
|
||||
if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
|
||||
# for target which must generate AutoGen code and makefile
|
||||
if not self.SkipAutoGen or Target == 'genc':
|
||||
self.Progress.Start("Generating code")
|
||||
AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
|
||||
self.Progress.Stop("done!")
|
||||
if Target == "genc":
|
||||
return True
|
||||
mqueue = mp.Queue()
|
||||
for m in AutoGenObject.GetAllModuleInfo:
|
||||
mqueue.put(m)
|
||||
|
||||
if not self.SkipAutoGen or Target == 'genmake':
|
||||
self.Progress.Start("Generating makefile")
|
||||
AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand)
|
||||
AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}
|
||||
self.Progress.Start("Generating makefile and code")
|
||||
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
|
||||
AutoGenObject.DataPipe.dump(data_pipe_file)
|
||||
autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList)
|
||||
self.Progress.Stop("done!")
|
||||
if Target == "genmake":
|
||||
return True
|
||||
else:
|
||||
# always recreate top/platform makefile when clean, just in case of inconsistency
|
||||
if not autogen_rt:
|
||||
self.AutoGenMgr.TerminateWorkers()
|
||||
self.AutoGenMgr.join(0.1)
|
||||
raise FatalError(errorcode)
|
||||
AutoGenObject.CreateCodeFile(False)
|
||||
AutoGenObject.CreateMakeFile(False)
|
||||
else:
|
||||
# always recreate top/platform makefile when clean, just in case of inconsistency
|
||||
AutoGenObject.CreateCodeFile(True)
|
||||
AutoGenObject.CreateMakeFile(True)
|
||||
|
||||
if EdkLogger.GetLevel() == EdkLogger.QUIET:
|
||||
EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
|
||||
|
|
@ -1334,8 +1362,8 @@ class Build():
|
|||
return True
|
||||
else:
|
||||
# always recreate top/platform makefile when clean, just in case of inconsistency
|
||||
AutoGenObject.CreateCodeFile(False)
|
||||
AutoGenObject.CreateMakeFile(False)
|
||||
AutoGenObject.CreateCodeFile(True)
|
||||
AutoGenObject.CreateMakeFile(True)
|
||||
|
||||
if EdkLogger.GetLevel() == EdkLogger.QUIET:
|
||||
EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
|
||||
|
|
@ -1713,9 +1741,10 @@ class Build():
|
|||
continue
|
||||
if Ma.PcdIsDriver:
|
||||
Ma.PlatformInfo = Pa
|
||||
Ma.Workspace = Wa
|
||||
PcdMaList.append(Ma)
|
||||
self.BuildModules.append(Ma)
|
||||
self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict)
|
||||
self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
|
||||
|
||||
# Create MAP file when Load Fix Address is enabled.
|
||||
if self.Target in ["", "all", "fds"]:
|
||||
|
|
@ -1850,7 +1879,7 @@ class Build():
|
|||
MakeStart = time.time()
|
||||
for Ma in self.BuildModules:
|
||||
if not Ma.IsBinaryModule:
|
||||
Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))
|
||||
Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
|
||||
# Break build if any build thread has error
|
||||
if BuildTask.HasError():
|
||||
# we need a full version of makefile for platform
|
||||
|
|
@ -1980,7 +2009,7 @@ class Build():
|
|||
Wa.CreateMakeFile(False)
|
||||
|
||||
# Add ffs build to makefile
|
||||
CmdListDict = None
|
||||
CmdListDict = {}
|
||||
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
|
||||
CmdListDict = self._GenFfsCmd(Wa.ArchList)
|
||||
|
||||
|
|
@ -1988,6 +2017,7 @@ class Build():
|
|||
ExitFlag = threading.Event()
|
||||
ExitFlag.clear()
|
||||
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
|
||||
self.BuildModules = []
|
||||
for Arch in Wa.ArchList:
|
||||
PcdMaList = []
|
||||
AutoGenStart = time.time()
|
||||
|
|
@ -2005,6 +2035,8 @@ class Build():
|
|||
if Inf in Pa.Platform.Modules:
|
||||
continue
|
||||
ModuleList.append(Inf)
|
||||
Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
|
||||
Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
|
||||
for Module in ModuleList:
|
||||
# Get ModuleAutoGen object to generate C code file and makefile
|
||||
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
|
||||
|
|
@ -2013,6 +2045,7 @@ class Build():
|
|||
continue
|
||||
if Ma.PcdIsDriver:
|
||||
Ma.PlatformInfo = Pa
|
||||
Ma.Workspace = Wa
|
||||
PcdMaList.append(Ma)
|
||||
if Ma.CanSkipbyHash():
|
||||
self.HashSkipModules.append(Ma)
|
||||
|
|
@ -2024,34 +2057,32 @@ class Build():
|
|||
EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
|
||||
|
||||
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
|
||||
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
|
||||
# for target which must generate AutoGen code and makefile
|
||||
if not self.SkipAutoGen or self.Target == 'genc':
|
||||
Ma.CreateCodeFile(True)
|
||||
if self.Target == "genc":
|
||||
continue
|
||||
|
||||
if not self.SkipAutoGen or self.Target == 'genmake':
|
||||
if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:
|
||||
Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])
|
||||
del CmdListDict[Module.File, Arch]
|
||||
else:
|
||||
Ma.CreateMakeFile(True)
|
||||
if self.Target == "genmake":
|
||||
continue
|
||||
self.BuildModules.append(Ma)
|
||||
# Initialize all modules in tracking to 'FAIL'
|
||||
if Ma.Arch not in GlobalData.gModuleBuildTracking:
|
||||
GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
|
||||
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
|
||||
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
|
||||
mqueue = mp.Queue()
|
||||
for m in Pa.GetAllModuleInfo:
|
||||
mqueue.put(m)
|
||||
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
|
||||
Pa.DataPipe.dump(data_pipe_file)
|
||||
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList)
|
||||
self.Progress.Stop("done!")
|
||||
self.AutoGenTime += int(round((time.time() - AutoGenStart)))
|
||||
if not autogen_rt:
|
||||
self.AutoGenMgr.TerminateWorkers()
|
||||
self.AutoGenMgr.join(0.1)
|
||||
raise FatalError(errorcode)
|
||||
for Arch in Wa.ArchList:
|
||||
MakeStart = time.time()
|
||||
for Ma in self.BuildModules:
|
||||
# Generate build task for the module
|
||||
if not Ma.IsBinaryModule:
|
||||
Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))
|
||||
Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
|
||||
# Break build if any build thread has error
|
||||
if BuildTask.HasError():
|
||||
# we need a full version of makefile for platform
|
||||
|
|
@ -2498,6 +2529,10 @@ def Main():
|
|||
EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
|
||||
ReturnCode = FORMAT_INVALID
|
||||
except KeyboardInterrupt:
|
||||
if MyBuild is not None:
|
||||
|
||||
# for multi-thread build exits safely
|
||||
MyBuild.Relinquish()
|
||||
ReturnCode = ABORT_ERROR
|
||||
if Option is not None and Option.debug is not None:
|
||||
EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
|
||||
|
|
@ -2554,6 +2589,10 @@ def Main():
|
|||
return ReturnCode
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
mp.set_start_method('spawn')
|
||||
except:
|
||||
pass
|
||||
r = Main()
|
||||
## 0-127 is a safe return range, and 1 is a standard default error
|
||||
if r < 0 or r > 127: r = 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue