BaseTools/Source/Python: New Target/ToolChain/Arch in DSC [BuildOptions]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=3359

Update BaseTools to support new build targets, new tool chains,
and new architectures declared in DSC file [BuildOptions] sections.

* Do not expand * when tools_def.txt is parsed.  Only expand when
  both tools_def.txt and DSC [BuilsOptions] sections have been parsed.
  This also requires more flexible matching of tool keys that contain *
  in tool key fields.

* Pre-scan the platform DSC file for FAMILY and TOOLCHAIN declarations
  DSC in [BuildOptions] sections before the FAMILY and TOOLCHAIN need
  to be known.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Michael D Kinney 2021-04-20 23:12:51 -07:00 committed by mergify[bot]
parent 612edbe6cd
commit ab957f036f
5 changed files with 230 additions and 96 deletions

View File

@ -173,17 +173,30 @@ class AutoGenInfo(object):
Family = Key[0]
Target, Tag, Arch, Tool, Attr = Key[1].split("_")
# if tool chain family doesn't match, skip it
if Tool in ToolDef and Family != "":
FamilyIsNull = False
if ToolDef[Tool].get(TAB_TOD_DEFINES_BUILDRULEFAMILY, "") != "":
if Family != ToolDef[Tool][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
continue
else:
if ToolDef[Tool].get(TAB_TOD_DEFINES_FAMILY, "") == "":
continue
if Family != ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
continue
FamilyMatch = True
if Family != "":
Found = False
if Tool in ToolDef:
FamilyIsNull = False
if TAB_TOD_DEFINES_BUILDRULEFAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
FamilyMatch = True
Found = True
if TAB_TOD_DEFINES_FAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
FamilyMatch = True
Found = True
if TAB_STAR in ToolDef:
FamilyIsNull = False
if TAB_TOD_DEFINES_BUILDRULEFAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
FamilyMatch = True
Found = True
if TAB_TOD_DEFINES_FAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_FAMILY]:
FamilyMatch = True
Found = True
if not Found:
continue
# expand any wildcard
if Target == TAB_STAR or Target == self.BuildTarget:
if Tag == TAB_STAR or Tag == self.ToolChain:
@ -213,12 +226,19 @@ class AutoGenInfo(object):
Family = Key[0]
Target, Tag, Arch, Tool, Attr = Key[1].split("_")
# if tool chain family doesn't match, skip it
if Tool not in ToolDef or Family == "":
if Family == "":
continue
# option has been added before
if TAB_TOD_DEFINES_FAMILY not in ToolDef[Tool]:
continue
if Family != ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
Found = False
if Tool in ToolDef:
if TAB_TOD_DEFINES_FAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
Found = True
if TAB_STAR in ToolDef:
if TAB_TOD_DEFINES_FAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_FAMILY]:
Found = True
if not Found:
continue
# expand any wildcard

View File

@ -827,6 +827,33 @@ class PlatformAutoGen(AutoGen):
RetVal = RetVal + _SplitOption(Flags.strip())
return RetVal
## Compute a tool defintion key priority value in range 0..15
#
# TARGET_TOOLCHAIN_ARCH_COMMANDTYPE_ATTRIBUTE 15
# ******_TOOLCHAIN_ARCH_COMMANDTYPE_ATTRIBUTE 14
# TARGET_*********_ARCH_COMMANDTYPE_ATTRIBUTE 13
# ******_*********_ARCH_COMMANDTYPE_ATTRIBUTE 12
# TARGET_TOOLCHAIN_****_COMMANDTYPE_ATTRIBUTE 11
# ******_TOOLCHAIN_****_COMMANDTYPE_ATTRIBUTE 10
# TARGET_*********_****_COMMANDTYPE_ATTRIBUTE 9
# ******_*********_****_COMMANDTYPE_ATTRIBUTE 8
# TARGET_TOOLCHAIN_ARCH_***********_ATTRIBUTE 7
# ******_TOOLCHAIN_ARCH_***********_ATTRIBUTE 6
# TARGET_*********_ARCH_***********_ATTRIBUTE 5
# ******_*********_ARCH_***********_ATTRIBUTE 4
# TARGET_TOOLCHAIN_****_***********_ATTRIBUTE 3
# ******_TOOLCHAIN_****_***********_ATTRIBUTE 2
# TARGET_*********_****_***********_ATTRIBUTE 1
# ******_*********_****_***********_ATTRIBUTE 0
#
def ToolDefinitionPriority (self,Key):
KeyList = Key.split('_')
Priority = 0
for Index in range (0, min(4, len(KeyList))):
if KeyList[Index] != '*':
Priority += (1 << Index)
return Priority
## Get tool chain definition
#
# Get each tool definition for given tool chain from tools_def.txt and platform
@ -839,8 +866,16 @@ class PlatformAutoGen(AutoGen):
ExtraData="[%s]" % self.MetaFile)
RetVal = OrderedDict()
DllPathList = set()
for Def in ToolDefinition:
PrioritizedDefList = sorted(ToolDefinition.keys(), key=self.ToolDefinitionPriority, reverse=True)
for Def in PrioritizedDefList:
Target, Tag, Arch, Tool, Attr = Def.split("_")
if Target == TAB_STAR:
Target = self.BuildTarget
if Tag == TAB_STAR:
Tag = self.ToolChain
if Arch == TAB_STAR:
Arch = self.Arch
if Target != self.BuildTarget or Tag != self.ToolChain or Arch != self.Arch:
continue
@ -850,9 +885,14 @@ class PlatformAutoGen(AutoGen):
DllPathList.add(Value)
continue
#
# ToolDefinition is sorted from highest priority to lowest priority.
# Only add the first(highest priority) match to RetVal
#
if Tool not in RetVal:
RetVal[Tool] = OrderedDict()
RetVal[Tool][Attr] = Value
if Attr not in RetVal[Tool]:
RetVal[Tool][Attr] = Value
ToolsDef = ''
if GlobalData.gOptions.SilentMode and "MAKE" in RetVal:
@ -860,9 +900,21 @@ class PlatformAutoGen(AutoGen):
RetVal["MAKE"]["FLAGS"] = ""
RetVal["MAKE"]["FLAGS"] += " -s"
MakeFlags = ''
for Tool in RetVal:
for Attr in RetVal[Tool]:
Value = RetVal[Tool][Attr]
ToolList = list(RetVal.keys())
ToolList.sort()
for Tool in ToolList:
if Tool == TAB_STAR:
continue
AttrList = list(RetVal[Tool].keys())
if TAB_STAR in ToolList:
AttrList += list(RetVal[TAB_STAR])
AttrList.sort()
for Attr in AttrList:
if Attr in RetVal[Tool]:
Value = RetVal[Tool][Attr]
else:
Value = RetVal[TAB_STAR][Attr]
if Tool in self._BuildOptionWithToolDef(RetVal) and Attr in self._BuildOptionWithToolDef(RetVal)[Tool]:
# check if override is indicated
if self._BuildOptionWithToolDef(RetVal)[Tool][Attr].startswith('='):
@ -877,7 +929,7 @@ class PlatformAutoGen(AutoGen):
if Attr == "PATH":
# Don't put MAKE definition in the file
if Tool != "MAKE":
ToolsDef += "%s = %s\n" % (Tool, Value)
ToolsDef += "%s_%s = %s\n" % (Tool, Attr, Value)
elif Attr != "DLL":
# Don't put MAKE definition in the file
if Tool == "MAKE":
@ -1469,17 +1521,31 @@ class PlatformAutoGen(AutoGen):
Family = Key[0]
Target, Tag, Arch, Tool, Attr = Key[1].split("_")
# if tool chain family doesn't match, skip it
if Tool in ToolDef and Family != "":
FamilyIsNull = False
if ToolDef[Tool].get(TAB_TOD_DEFINES_BUILDRULEFAMILY, "") != "":
if Family != ToolDef[Tool][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
continue
else:
if ToolDef[Tool].get(TAB_TOD_DEFINES_FAMILY, "") == "":
continue
if Family != ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
continue
FamilyMatch = True
if Family != "":
Found = False
if Tool in ToolDef:
FamilyIsNull = False
if TAB_TOD_DEFINES_BUILDRULEFAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
FamilyMatch = True
Found = True
if TAB_TOD_DEFINES_FAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
FamilyMatch = True
Found = True
if TAB_STAR in ToolDef:
FamilyIsNull = False
if TAB_TOD_DEFINES_BUILDRULEFAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_BUILDRULEFAMILY]:
FamilyMatch = True
Found = True
if TAB_TOD_DEFINES_FAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_FAMILY]:
FamilyMatch = True
Found = True
if not Found:
continue
# expand any wildcard
if Target == TAB_STAR or Target == self.BuildTarget:
if Tag == TAB_STAR or Tag == self.ToolChain:
@ -1509,12 +1575,19 @@ class PlatformAutoGen(AutoGen):
Family = Key[0]
Target, Tag, Arch, Tool, Attr = Key[1].split("_")
# if tool chain family doesn't match, skip it
if Tool not in ToolDef or Family == "":
if Family == "":
continue
# option has been added before
if TAB_TOD_DEFINES_FAMILY not in ToolDef[Tool]:
continue
if Family != ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
Found = False
if Tool in ToolDef:
if TAB_TOD_DEFINES_FAMILY in ToolDef[Tool]:
if Family == ToolDef[Tool][TAB_TOD_DEFINES_FAMILY]:
Found = True
if TAB_STAR in ToolDef:
if TAB_TOD_DEFINES_FAMILY in ToolDef[TAB_STAR]:
if Family == ToolDef[TAB_STAR][TAB_TOD_DEFINES_FAMILY]:
Found = True
if not Found:
continue
# expand any wildcard

View File

@ -1,7 +1,7 @@
## @file
# This file is used to define each component of tools_def.txt file
#
# Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
@ -86,23 +86,6 @@ class ToolDefClassObject(object):
self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
for Index in range(3, -1, -1):
# make a copy of the keys to enumerate over to prevent issues when
# adding/removing items from the original dict.
for Key in list(self.ToolsDefTxtDictionary.keys()):
List = Key.split('_')
if List[Index] == TAB_STAR:
for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
List[Index] = String
NewKey = '%s_%s_%s_%s_%s' % tuple(List)
if NewKey not in self.ToolsDefTxtDictionary:
self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
del self.ToolsDefTxtDictionary[Key]
elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
del self.ToolsDefTxtDictionary[Key]
## IncludeToolDefFile
#
# Load target.txt file and parse it as if its contents were inside the main file

View File

@ -1,7 +1,7 @@
## @file
# Global variables for GenFds
#
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
@ -875,14 +875,27 @@ def FindExtendTool(KeyStringList, CurrentArchList, NameGuid):
ToolOptionKey = None
KeyList = None
for tool_def in ToolDefinition.items():
if NameGuid.lower() == tool_def[1].lower():
KeyList = tool_def[0].split('_')
Key = KeyList[0] + \
'_' + \
KeyList[1] + \
'_' + \
KeyList[2]
if Key in KeyStringList and KeyList[4] == DataType.TAB_GUID:
KeyList = tool_def[0].split('_')
if len(KeyList) < 5:
continue
if KeyList[4] != DataType.TAB_GUID:
continue
if NameGuid.lower() != tool_def[1].lower():
continue
Key = KeyList[0] + \
'_' + \
KeyList[1] + \
'_' + \
KeyList[2]
for KeyString in KeyStringList:
KeyStringBuildTarget, KeyStringToolChain, KeyStringArch = KeyString.split('_')
if KeyList[0] == DataType.TAB_STAR:
KeyList[0] = KeyStringBuildTarget
if KeyList[1] == DataType.TAB_STAR:
KeyList[1] = KeyStringToolChain
if KeyList[2] == DataType.TAB_STAR:
KeyList[2] = KeyStringArch
if KeyList[0] == KeyStringBuildTarget and KeyList[1] == KeyStringToolChain and KeyList[2] == KeyStringArch:
ToolPathKey = Key + '_' + KeyList[3] + '_PATH'
ToolOptionKey = Key + '_' + KeyList[3] + '_FLAGS'
ToolPath = ToolDefinition.get(ToolPathKey)

View File

@ -2,7 +2,7 @@
# build a platform or a module
#
# Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
# Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>
# Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
# Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
#
@ -889,6 +889,47 @@ class Build():
except:
return False, UNKNOWN_ERROR
## Add TOOLCHAIN and FAMILY declared in DSC [BuildOptions] to ToolsDefTxtDatabase.
#
# Loop through the set of build targets, tool chains, and archs provided on either
# the command line or in target.txt to discover FAMILY and TOOLCHAIN delclarations
# in [BuildOptions] sections that may be within !if expressions that may use
# $(TARGET), $(TOOLCHAIN), $(TOOLCHAIN_TAG), or $(ARCH) operands.
#
def GetToolChainAndFamilyFromDsc (self, File):
for BuildTarget in self.BuildTargetList:
GlobalData.gGlobalDefines['TARGET'] = BuildTarget
for BuildToolChain in self.ToolChainList:
GlobalData.gGlobalDefines['TOOLCHAIN'] = BuildToolChain
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = BuildToolChain
for BuildArch in self.ArchList:
GlobalData.gGlobalDefines['ARCH'] = BuildArch
dscobj = self.BuildDatabase[File, BuildArch]
for KeyFamily, Key, KeyCodeBase in dscobj.BuildOptions:
try:
Target, ToolChain, Arch, Tool, Attr = Key.split('_')
except:
continue
if ToolChain == TAB_STAR or Attr != TAB_TOD_DEFINES_FAMILY:
continue
try:
Family = dscobj.BuildOptions[(KeyFamily, Key, KeyCodeBase)]
Family = Family.strip().strip('=').strip()
except:
continue
if TAB_TOD_DEFINES_FAMILY not in self.ToolDef.ToolsDefTxtDatabase:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][ToolChain] = Family
if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolDef.ToolsDefTxtDatabase:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY]:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][ToolChain] = Family
if TAB_TOD_DEFINES_TOOL_CHAIN_TAG not in self.ToolDef.ToolsDefTxtDatabase:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = []
if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].append(ToolChain)
## Load configuration
#
# This method will parse target.txt and get the build configurations.
@ -910,6 +951,26 @@ class Build():
if self.ToolChainList is None or len(self.ToolChainList) == 0:
EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
if not self.PlatformFile:
PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
if not PlatformFile:
# Try to find one in current directory
WorkingDirectory = os.getcwd()
FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
FileNum = len(FileList)
if FileNum >= 2:
EdkLogger.error("build", OPTION_MISSING,
ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
elif FileNum == 1:
PlatformFile = FileList[0]
else:
EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
self.GetToolChainAndFamilyFromDsc (self.PlatformFile)
# check if the tool chains are defined or not
NewToolChainList = []
for ToolChain in self.ToolChainList:
@ -935,23 +996,6 @@ class Build():
ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
self.ToolChainFamily = ToolChainFamily
if not self.PlatformFile:
PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
if not PlatformFile:
# Try to find one in current directory
WorkingDirectory = os.getcwd()
FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
FileNum = len(FileList)
if FileNum >= 2:
EdkLogger.error("build", OPTION_MISSING,
ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
elif FileNum == 1:
PlatformFile = FileList[0]
else:
EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
self.ThreadNumber = ThreadNum()
## Initialize build configuration
#
@ -2381,24 +2425,25 @@ class Build():
continue
for Arch in self.ArchList:
# Build up the list of supported architectures for this build
prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
# Look through the tool definitions for GUIDed tools
guidAttribs = []
for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():
if attrib.upper().endswith('_GUID'):
split = attrib.split('_')
thisPrefix = '_'.join(split[0:3]) + '_'
if thisPrefix == prefix:
guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
guid = guid.lower()
toolName = split[3]
path = '_'.join(split[0:4]) + '_PATH'
path = self.ToolDef.ToolsDefTxtDictionary[path]
path = self.GetRealPathOfTool(path)
guidAttribs.append((guid, toolName, path))
GuidBuildTarget, GuidToolChain, GuidArch, GuidTool, GuidAttr = attrib.split('_')
if GuidAttr.upper() == 'GUID':
if GuidBuildTarget == TAB_STAR:
GuidBuildTarget = BuildTarget
if GuidToolChain == TAB_STAR:
GuidToolChain = ToolChain
if GuidArch == TAB_STAR:
GuidArch = Arch
if GuidBuildTarget == BuildTarget and GuidToolChain == ToolChain and GuidArch == Arch:
path = '_'.join(attrib.split('_')[:-1]) + '_PATH'
if path in self.ToolDef.ToolsDefTxtDictionary:
path = self.ToolDef.ToolsDefTxtDictionary[path]
path = self.GetRealPathOfTool(path)
guidAttribs.append((value.lower(), GuidTool, path))
# Sort by GuidTool name
sorted (guidAttribs, key=lambda x: x[1])
# Write out GuidedSecTools.txt
toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
toolsFile = open(toolsFile, 'wt')