BaseTools: Fix nmake failure due to command-line length limitation

NMAKE is limited to command-line length of 4096 characters. Due to the
large number of /I directives specified on command line (one per include
directory), the path length of WORKSPACE is multiplied by the number of
/I directives and can exceed the limit.
This patch:
1. Add new build option -l, --cmd-len to set the maximum command line
length, default value is 4096.
2. Generate the response file only if the command line length exceed its
maximum characters (default is 4096) when build the module. Cover
PP_FLAGS, CC_FLAGS, VFRPP_FLAGS, APP_FLAGS, ASLPP_FLAGS, ASLCC_FLAGS and
ASM_FLAGS.
3. The content of the response file is combine from the FLAGS option and
INC option.
4. When build failure, it would print out the response file's file
location and its content.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Yonghong Zhu <yonghong.zhu@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Yonghong Zhu 2016-03-16 11:06:44 +08:00
parent 3362c5f17a
commit 725cdb8fbf
4 changed files with 135 additions and 2 deletions

View File

@ -2378,6 +2378,7 @@ class ModuleAutoGen(AutoGen):
self._MakeFileDir = None self._MakeFileDir = None
self._IncludePathList = None self._IncludePathList = None
self._IncludePathLength = 0
self._AutoGenFileList = None self._AutoGenFileList = None
self._UnicodeFileList = None self._UnicodeFileList = None
self._SourceFileList = None self._SourceFileList = None
@ -3224,6 +3225,13 @@ class ModuleAutoGen(AutoGen):
self._IncludePathList.append(str(Inc)) self._IncludePathList.append(str(Inc))
return self._IncludePathList return self._IncludePathList
def _GetIncludePathLength(self):
self._IncludePathLength = 0
if self._IncludePathList:
for inc in self._IncludePathList:
self._IncludePathLength += len(' ' + inc)
return self._IncludePathLength
## Get HII EX PCDs which maybe used by VFR ## Get HII EX PCDs which maybe used by VFR
# #
# efivarstore used by VFR may relate with HII EX PCDs # efivarstore used by VFR may relate with HII EX PCDs
@ -3816,6 +3824,7 @@ class ModuleAutoGen(AutoGen):
CustomMakefile = property(_GetCustomMakefile) CustomMakefile = property(_GetCustomMakefile)
IncludePathList = property(_GetIncludePathList) IncludePathList = property(_GetIncludePathList)
IncludePathLength = property(_GetIncludePathLength)
AutoGenFileList = property(_GetAutoGenFileList) AutoGenFileList = property(_GetAutoGenFileList)
UnicodeFileList = property(_GetUnicodeFileList) UnicodeFileList = property(_GetUnicodeFileList)
SourceFileList = property(_GetSourceFileList) SourceFileList = property(_GetSourceFileList)

View File

@ -1,7 +1,7 @@
## @file ## @file
# Create makefile for MS nmake and GNU make # Create makefile for MS nmake and GNU make
# #
# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR> # Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR>
# This program and the accompanying materials # This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License # are licensed and made available under the terms and conditions of the BSD License
# which accompanies this distribution. The full text of the license may be found at # which accompanies this distribution. The full text of the license may be found at
@ -497,6 +497,22 @@ cleanlib:
ToolsDef.append("%s_%s = %s" % (Tool, Attr, Value)) ToolsDef.append("%s_%s = %s" % (Tool, Attr, Value))
ToolsDef.append("") ToolsDef.append("")
# generate the Response file and Response flag
RespDict = self.CommandExceedLimit()
RespFileList = os.path.join(self._AutoGenObject.OutputDir, 'respfilelist.txt')
if RespDict:
RespFileListContent = ''
for Resp in RespDict.keys():
RespFile = os.path.join(self._AutoGenObject.OutputDir, str(Resp).lower() + '.txt')
SaveFileOnChange(RespFile, RespDict[Resp], False)
ToolsDef.append("%s = %s" % (Resp, '@' + RespFile))
RespFileListContent += '@' + RespFile + os.linesep
RespFileListContent += RespDict[Resp] + os.linesep
SaveFileOnChange(RespFileList, RespFileListContent, False)
else:
if os.path.exists(RespFileList):
os.remove(RespFileList)
# convert source files and binary files to build targets # convert source files and binary files to build targets
self.ResultFileList = [str(T.Target) for T in self._AutoGenObject.CodaTargetList] self.ResultFileList = [str(T.Target) for T in self._AutoGenObject.CodaTargetList]
if len(self.ResultFileList) == 0 and len(self._AutoGenObject.SourceFileList) <> 0: if len(self.ResultFileList) == 0 and len(self._AutoGenObject.SourceFileList) <> 0:
@ -620,6 +636,102 @@ cleanlib:
return MakefileTemplateDict return MakefileTemplateDict
def CommandExceedLimit(self):
FlagDict = {
'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False},
'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False},
'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False},
'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False},
'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False},
'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False},
'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False},
}
RespDict = {}
FileTypeList = []
IncPrefix = self._INC_FLAG_[self._AutoGenObject.ToolChainFamily]
# base on the source files to decide the file type
for File in self._AutoGenObject.SourceFileList:
for type in self._AutoGenObject.FileTypes:
if File in self._AutoGenObject.FileTypes[type]:
if type not in FileTypeList:
FileTypeList.append(type)
# calculate the command-line length
if FileTypeList:
for type in FileTypeList:
BuildTargets = self._AutoGenObject.BuildRules[type].BuildTargets
for Target in BuildTargets:
CommandList = BuildTargets[Target].Commands
for SingleCommand in CommandList:
Tool = ''
SingleCommandLength = len(SingleCommand)
SingleCommandList = SingleCommand.split()
if len(SingleCommandList) > 0:
for Flag in FlagDict.keys():
if '$('+ Flag +')' in SingleCommandList[0]:
Tool = Flag
break
if Tool:
SingleCommandLength += len(self._AutoGenObject._BuildOption[Tool]['PATH'])
for item in SingleCommandList[1:]:
if FlagDict[Tool]['Macro'] in item:
Str = self._AutoGenObject._BuildOption[Tool]['FLAGS']
while(Str.find('$(') != -1):
for macro in self._AutoGenObject.Macros.keys():
MacroName = '$('+ macro + ')'
if (Str.find(MacroName) != -1):
Str = Str.replace(MacroName, self._AutoGenObject.Macros[macro])
break
else:
EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))
SingleCommandLength += len(Str)
elif '$(INC)' in item:
SingleCommandLength += self._AutoGenObject.IncludePathLength + len(IncPrefix) * len(self._AutoGenObject._IncludePathList)
elif item.find('$(') != -1:
Str = item
for Option in self._AutoGenObject.BuildOption.keys():
for Attr in self._AutoGenObject.BuildOption[Option]:
if Str.find(Option + '_' + Attr) != -1:
Str = Str.replace('$(' + Option + '_' + Attr + ')', self._AutoGenObject.BuildOption[Option][Attr])
while(Str.find('$(') != -1):
for macro in self._AutoGenObject.Macros.keys():
MacroName = '$('+ macro + ')'
if (Str.find(MacroName) != -1):
Str = Str.replace(MacroName, self._AutoGenObject.Macros[macro])
break
else:
EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))
SingleCommandLength += len(Str)
if SingleCommandLength > GlobalData.gCommandMaxLength:
FlagDict[Tool]['Value'] = True
# generate the response file content by combine the FLAGS and INC
for Flag in FlagDict.keys():
if FlagDict[Flag]['Value']:
Key = Flag + '_RESP'
RespMacro = FlagDict[Flag]['Macro'].replace('FLAGS', 'RESP')
Value = self._AutoGenObject.BuildOption[Flag]['FLAGS']
for inc in self._AutoGenObject._IncludePathList:
Value += ' ' + IncPrefix + inc
while (Value.find('$(') != -1):
for macro in self._AutoGenObject.Macros.keys():
MacroName = '$('+ macro + ')'
if (Value.find(MacroName) != -1):
Value = Value.replace(MacroName, self._AutoGenObject.Macros[macro])
break
else:
EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))
RespDict[Key] = Value
for Target in BuildTargets:
for i, SingleCommand in enumerate(BuildTargets[Target].Commands):
if FlagDict[Flag]['Macro'] in SingleCommand:
BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)','').replace(FlagDict[Flag]['Macro'], RespMacro)
return RespDict
def ProcessBuildTargetList(self): def ProcessBuildTargetList(self):
# #
# Search dependency file list for each source file # Search dependency file list for each source file

View File

@ -34,7 +34,7 @@ gActivePlatform = None
gCommandLineDefines = {} gCommandLineDefines = {}
gEdkGlobal = {} gEdkGlobal = {}
gOverrideDir = {} gOverrideDir = {}
gCommandMaxLength = 4096
# for debug trace purpose when problem occurs # for debug trace purpose when problem occurs
gProcessingFile = '' gProcessingFile = ''
gBuildingModule = '' gBuildingModule = ''

View File

@ -304,6 +304,14 @@ def LaunchCommand(Command, WorkingDir):
if Proc.returncode != 0: if Proc.returncode != 0:
if type(Command) != type(""): if type(Command) != type(""):
Command = " ".join(Command) Command = " ".join(Command)
# print out the Response file and its content when make failure
RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
if os.path.isfile(RespFile):
f = open(RespFile)
RespContent = f.read()
f.close()
EdkLogger.info(RespContent)
EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir)) EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
## The smallest unit that can be built in multi-thread build mode ## The smallest unit that can be built in multi-thread build mode
@ -775,6 +783,9 @@ class Build():
self.UniFlag = BuildOptions.Flag self.UniFlag = BuildOptions.Flag
self.BuildModules = [] self.BuildModules = []
if BuildOptions.CommandLength:
GlobalData.gCommandMaxLength = BuildOptions.CommandLength
# print dot character during doing some time-consuming work # print dot character during doing some time-consuming work
self.Progress = Utils.Progressor() self.Progress = Utils.Progressor()
@ -1931,6 +1942,7 @@ def MyOptionParser():
Parser.add_option("--check-usage", action="store_true", dest="CheckUsage", default=False, help="Check usage content of entries listed in INF file.") Parser.add_option("--check-usage", action="store_true", dest="CheckUsage", default=False, help="Check usage content of entries listed in INF file.")
Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files") Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")
Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: 'PcdName=Value' ") Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: 'PcdName=Value' ")
Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")
(Opt, Args) = Parser.parse_args() (Opt, Args) = Parser.parse_args()
return (Opt, Args) return (Opt, Args)