tyler.durden
/
audk
mirror of https://github.com/acidanthera/audk.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BaseTools: Improve the method of checking queue empty 

Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807

The Queue.empty() method is not reliable in the multiple
process runtime environment. This patch uses a new method
to check if all modules are processed and workers need
to be stopped. That is to add a None item at the bottom
of the queue. Worker check if it gets that None item to
know if all the module is processed.

Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Cc: Lucy Yan <lucyyan@google.com>

Reviewed-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Feng, Bob C 2020-08-03 10:03:38 +08:00 committed by mergify[bot]
parent dcf51c05e8
commit bfe36cb4ef
2 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
BaseTools/Source/Python/AutoGen/AutoGenWorker.py
View File

@ -24,6 +24,7 @@ import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe
import logging
import time
def clearQ(q):
try:
@ -111,7 +112,11 @@ class AutoGenManager(threading.Thread):
break
if badnews == "Done":
fin_num += 1
elif badnews == "QueueEmpty":
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
self.TerminateWorkers()
else:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
self.Status = False
self.TerminateWorkers()
if fin_num == len(self.autogen_workers):
@ -227,12 +232,21 @@ class AutoGenWorkerInProcess(mp.Process):
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
if self.module_queue.empty():
break
if self.error_event.is_set():
break
module_count += 1
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
try:
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
except Empty:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))
time.sleep(0.01)
continue
if module_file is None:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))
self.feedback_q.put("QueueEmpty")
time.sleep(0.01)
continue
modulefullpath = os.path.join(module_root,module_file)
taskname = " : ".join((modulefullpath,module_arch))
module_metafile = PathClass(module_file,module_root)
@ -280,11 +294,11 @@ class AutoGenWorkerInProcess(mp.Process):
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
except Empty:
pass
except:
except Exception as e:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))
self.feedback_q.put(taskname)
finally:
EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")

3
BaseTools/Source/Python/build/build.py
View File

@ -1217,7 +1217,7 @@ class Build():
mqueue = mp.Queue()
for m in AutoGenObject.GetAllModuleInfo:
mqueue.put(m)
mqueue.put((None,None,None,None,None,None,None))
AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
AutoGenObject.CreateLibModuelDirs()
@ -2174,6 +2174,7 @@ class Build():
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file)
mqueue.put((None,None,None,None,None,None,None))
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
if not autogen_rt: