mirror of https://github.com/acidanthera/audk.git
BaseTools: Support multi thread build Basetool on Windows
Add NmakeSubdirs.py to replace NmakeSubdirs.bat in VS Makefile. This script will
invoke nmake in multi thread mode. It can save more than half time of BaseTools
C clean build.
GCC make supports multiple thread in make phase. So, GNUmakefile doesn't need apply
this script.
single task or job=1:
just single thread and invoke subprocess,subprocess will use
system.stdout to print output.
multi task:
thread number is logic cpu count.All subprocess output will pass to
python script by PIPE and then script print it to system.stdout.
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Dongao Guo<dongao.guo@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
Test-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Dongao Guo
Liming Gao
parent
73dbd6afab
commit
4c0d19e5bf
|
|
@ -1,7 +1,7 @@
|
|||
## @file
|
||||
# Windows makefile for Base Tools project build.
|
||||
#
|
||||
# Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
|
||||
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
|
||||
# This program and the accompanying materials
|
||||
# are licensed and made available under the terms and conditions of the BSD License
|
||||
# which accompanies this distribution. The full text of the license may be found at
|
||||
|
|
@ -20,19 +20,19 @@ SUBDIRS = $(BASE_TOOLS_PATH)\Source\C $(BASE_TOOLS_PATH)\Source\Python
|
|||
all: c python
|
||||
|
||||
c :
|
||||
@$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\C
|
||||
@$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $(BASE_TOOLS_PATH)\Source\C
|
||||
|
||||
python:
|
||||
@$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $(BASE_TOOLS_PATH)\Source\Python
|
||||
@$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $(BASE_TOOLS_PATH)\Source\Python
|
||||
|
||||
subdirs: $(SUBDIRS)
|
||||
@$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat all $**
|
||||
@$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py all $**
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat clean $(SUBDIRS)
|
||||
$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py clean $(SUBDIRS)
|
||||
|
||||
.PHONY: cleanall
|
||||
cleanall:
|
||||
@$(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.bat cleanall $(SUBDIRS)
|
||||
$(PYTHON_HOME)\python.exe $(BASE_TOOLS_PATH)\Source\C\Makefiles\NmakeSubdirs.py cleanall $(SUBDIRS)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
## @file
|
||||
# Windows makefile for C tools build.
|
||||
#
|
||||
# Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.<BR>
|
||||
# Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
|
||||
# This program and the accompanying materials
|
||||
# are licensed and made available under the terms and conditions of the BSD License
|
||||
# which accompanies this distribution. The full text of the license may be found at
|
||||
|
|
@ -16,7 +16,7 @@ HOST_ARCH = IA32
|
|||
|
||||
LIBRARIES = Common
|
||||
APPLICATIONS = \
|
||||
BootSectImage \
|
||||
VfrCompile \
|
||||
BrotliCompress \
|
||||
EfiLdrImage \
|
||||
EfiRom \
|
||||
|
|
@ -32,7 +32,7 @@ APPLICATIONS = \
|
|||
Split \
|
||||
TianoCompress \
|
||||
VolInfo \
|
||||
VfrCompile \
|
||||
BootSectImage \
|
||||
DevicePath
|
||||
|
||||
all: libs apps install
|
||||
|
|
@ -43,7 +43,7 @@ libs: $(LIBRARIES)
|
|||
@echo # Build libraries
|
||||
@echo ######################
|
||||
@if not exist $(LIB_PATH) mkdir $(LIB_PATH)
|
||||
@Makefiles\NmakeSubdirs.bat all $**
|
||||
@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $**
|
||||
|
||||
apps: $(APPLICATIONS)
|
||||
@echo.
|
||||
|
|
@ -51,7 +51,7 @@ apps: $(APPLICATIONS)
|
|||
@echo # Build executables
|
||||
@echo ######################
|
||||
@if not exist $(BIN_PATH) mkdir $(BIN_PATH)
|
||||
@Makefiles\NmakeSubdirs.bat all $**
|
||||
@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py all $**
|
||||
|
||||
install: $(LIB_PATH) $(BIN_PATH)
|
||||
@echo.
|
||||
|
|
@ -65,11 +65,11 @@ install: $(LIB_PATH) $(BIN_PATH)
|
|||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@Makefiles\NmakeSubdirs.bat clean $(LIBRARIES) $(APPLICATIONS)
|
||||
@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py clean $(LIBRARIES) $(APPLICATIONS)
|
||||
|
||||
.PHONY: cleanall
|
||||
cleanall:
|
||||
@Makefiles\NmakeSubdirs.bat cleanall $(LIBRARIES) $(APPLICATIONS)
|
||||
@$(PYTHON_HOME)\python.exe Makefiles\NmakeSubdirs.py cleanall $(LIBRARIES) $(APPLICATIONS)
|
||||
|
||||
!INCLUDE Makefiles\ms.rule
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
# @file NmakeSubdirs.py
|
||||
# This script support parallel build for nmake in windows environment.
|
||||
# It supports Python2.x and Python3.x both.
|
||||
#
|
||||
# Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
|
||||
#
|
||||
# This program and the accompanying materials
|
||||
# are licensed and made available under the terms and conditions of the BSD License
|
||||
# which accompanies this distribution. The full text of the license may be found at
|
||||
# http://opensource.org/licenses/bsd-license.php
|
||||
#
|
||||
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
|
||||
#
|
||||
|
||||
#
|
||||
# Import Modules
|
||||
#
|
||||
|
||||
from __future__ import print_function
|
||||
import argparse
|
||||
import threading
|
||||
import time
|
||||
import os
|
||||
import subprocess
|
||||
import multiprocessing
|
||||
import copy
|
||||
import sys
|
||||
__prog__ = 'NmakeSubdirs'
|
||||
__version__ = '%s Version %s' % (__prog__, '0.10 ')
|
||||
__copyright__ = 'Copyright (c) 2018, Intel Corporation. All rights reserved.'
|
||||
__description__ = 'Replace for NmakeSubdirs.bat in windows ,support parallel build for nmake.\n'
|
||||
|
||||
cpu_count = multiprocessing.cpu_count()
|
||||
output_lock = threading.Lock()
|
||||
def RunCommand(WorkDir=None, *Args, **kwargs):
|
||||
if WorkDir is None:
|
||||
WorkDir = os.curdir
|
||||
if "stderr" not in kwargs:
|
||||
kwargs["stderr"] = subprocess.STDOUT
|
||||
if "stdout" not in kwargs:
|
||||
kwargs["stdout"] = subprocess.PIPE
|
||||
p = subprocess.Popen(Args, cwd=WorkDir, stderr=kwargs["stderr"], stdout=kwargs["stdout"])
|
||||
stdout, stderr = p.communicate()
|
||||
message = ""
|
||||
if stdout is not None:
|
||||
message = stdout.decode() #for compatibility in python 2 and 3
|
||||
|
||||
if p.returncode != 0:
|
||||
raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))
|
||||
|
||||
output_lock.acquire(True)
|
||||
print("execute command \"{0}\" in directory {1}".format(" ".join(Args), WorkDir))
|
||||
print(message)
|
||||
output_lock.release()
|
||||
|
||||
return p.returncode, stdout
|
||||
|
||||
class TaskUnit(object):
|
||||
def __init__(self, func, args, kwargs):
|
||||
self.func = func
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
def __eq__(self, other):
|
||||
return id(self).__eq__(id(other))
|
||||
|
||||
def run(self):
|
||||
return self.func(*self.args, **self.kwargs)
|
||||
|
||||
def __str__(self):
|
||||
para = list(self.args)
|
||||
para.extend("{0}={1}".format(k, v)for k, v in self.kwargs.items())
|
||||
|
||||
return "{0}({1})".format(self.func.__name__, ",".join(para))
|
||||
|
||||
class ThreadControl(object):
|
||||
|
||||
def __init__(self, maxthread):
|
||||
self._processNum = maxthread
|
||||
self.pending = []
|
||||
self.running = []
|
||||
self.pendingLock = threading.Lock()
|
||||
self.runningLock = threading.Lock()
|
||||
self.error = False
|
||||
self.errorLock = threading.Lock()
|
||||
self.errorMsg = "errorMsg"
|
||||
|
||||
def addTask(self, func, *args, **kwargs):
|
||||
self.pending.append(TaskUnit(func, args, kwargs))
|
||||
|
||||
def waitComplete(self):
|
||||
self._schedule.join()
|
||||
|
||||
def startSchedule(self):
|
||||
self._schedule = threading.Thread(target=self.Schedule)
|
||||
self._schedule.start()
|
||||
|
||||
def Schedule(self):
|
||||
for i in range(self._processNum):
|
||||
task = threading.Thread(target=self.startTask)
|
||||
task.daemon = False
|
||||
self.running.append(task)
|
||||
|
||||
self.runningLock.acquire(True)
|
||||
for thread in self.running:
|
||||
thread.start()
|
||||
self.runningLock.release()
|
||||
|
||||
while len(self.running) > 0:
|
||||
time.sleep(0.1)
|
||||
if self.error:
|
||||
print("subprocess not exit sucessfully")
|
||||
print(self.errorMsg)
|
||||
|
||||
def startTask(self):
|
||||
while True:
|
||||
if self.error:
|
||||
break
|
||||
self.pendingLock.acquire(True)
|
||||
if len(self.pending) == 0:
|
||||
self.pendingLock.release()
|
||||
break
|
||||
task = self.pending.pop(0)
|
||||
self.pendingLock.release()
|
||||
try:
|
||||
task.run()
|
||||
except RuntimeError as e:
|
||||
if self.error: break
|
||||
self.errorLock.acquire(True)
|
||||
self.error = True
|
||||
self.errorMsg = str(e)
|
||||
time.sleep(0.1)
|
||||
self.errorLock.release()
|
||||
break
|
||||
|
||||
self.runningLock.acquire(True)
|
||||
self.running.remove(threading.currentThread())
|
||||
self.runningLock.release()
|
||||
|
||||
def Run():
|
||||
curdir = os.path.abspath(os.curdir)
|
||||
if len(args.subdirs) == 1:
|
||||
args.jobs = 1
|
||||
if args.jobs == 1:
|
||||
try:
|
||||
for dir in args.subdirs:
|
||||
RunCommand(os.path.join(curdir, dir), "nmake", args.target, stdout=sys.stdout, stderr=subprocess.STDOUT)
|
||||
except RuntimeError:
|
||||
exit(1)
|
||||
else:
|
||||
controller = ThreadControl(args.jobs)
|
||||
for dir in args.subdirs:
|
||||
controller.addTask(RunCommand, os.path.join(curdir, dir), "nmake", args.target)
|
||||
controller.startSchedule()
|
||||
controller.waitComplete()
|
||||
if controller.error:
|
||||
exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(prog=__prog__, description=__description__ + __copyright__, conflict_handler='resolve')
|
||||
|
||||
parser.add_argument("target", help="the target for nmake")
|
||||
parser.add_argument("subdirs", nargs="+", help="the relative dir path of makefile")
|
||||
parser.add_argument("--jobs", type=int, dest="jobs", default=cpu_count, help="thread number")
|
||||
parser.add_argument('--version', action='version', version=__version__)
|
||||
args = parser.parse_args()
|
||||
Run()
|
||||
|
||||
Loading…
Reference in New Issue