From 35f613d96ce43c7b23cd77aab063424ec4422e0c Mon Sep 17 00:00:00 2001 From: Yunhua Feng Date: Sat, 27 Jan 2018 00:28:05 +0800 Subject: [PATCH] BaseTools: Enhance parse performance by optimize ValueExpressionEx Optimize ValueExpressionEx function to enhance meta-data file parse performance. Cc: Liming Gao Cc: Yonghong Zhu Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Yunhua Feng Reviewed-by: Yonghong Zhu Reviewed-by: Liming Gao --- BaseTools/Source/Python/AutoGen/AutoGen.py | 16 +- BaseTools/Source/Python/Common/Expression.py | 307 ++++++++++-------- .../Source/Python/Workspace/DscBuildData.py | 21 +- .../Source/Python/Workspace/MetaFileParser.py | 10 +- BaseTools/Source/Python/build/BuildReport.py | 9 +- 5 files changed, 184 insertions(+), 179 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index ab178c9a4a..1cf50e872f 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -1245,6 +1245,7 @@ class PlatformAutoGen(AutoGen): # get the original module/package/platform objects self.BuildDatabase = Workspace.BuildDatabase self.DscBuildDataObj = Workspace.Platform + self._GuidDict = Workspace._GuidDict # flag indicating if the makefile/C-code file has been created or not self.IsMakeFileCreated = False @@ -2463,22 +2464,9 @@ class PlatformAutoGen(AutoGen): if FromPcd.SkuInfoList not in [None, '', []]: ToPcd.SkuInfoList = FromPcd.SkuInfoList # Add Flexible PCD format parse - PcdValue = ToPcd.DefaultValue - if PcdValue: - try: - ToPcd.DefaultValue = ValueExpression(PcdValue)(True) - except WrnExpression, Value: - ToPcd.DefaultValue = Value.result - except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' %(ToPcd.TokenSpaceGuidCName, ToPcd.TokenCName, ToPcd.DefaultValue, Value), - File=self.MetaFile) if ToPcd.DefaultValue: - _GuidDict = {} - for Pkg in self.PackageList: - Guids = Pkg.Guids - _GuidDict.update(Guids) try: - ToPcd.DefaultValue = ValueExpressionEx(ToPcd.DefaultValue, ToPcd.DatumType, _GuidDict)(True) + ToPcd.DefaultValue = ValueExpressionEx(ToPcd.DefaultValue, ToPcd.DatumType, self._GuidDict)(True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' %(ToPcd.TokenSpaceGuidCName, ToPcd.TokenCName, ToPcd.DefaultValue, Value), File=self.MetaFile) diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py index 55fa06d414..486c537029 100644 --- a/BaseTools/Source/Python/Common/Expression.py +++ b/BaseTools/Source/Python/Common/Expression.py @@ -1,7 +1,7 @@ ## @file # This file is used to parse and evaluate expression in directive or PCD value. # -# Copyright (c) 2011 - 2017, Intel Corporation. All rights reserved.
+# Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.
# This program and the accompanying materials # are licensed and made available under the terms and conditions of the BSD License # which accompanies this distribution. The full text of the license may be found at @@ -251,9 +251,6 @@ class ValueExpression(object): self._Expr = Expression self._NoProcess = True return - if Expression.strip().startswith('{') and Expression.strip().endswith('}'): - self._Expr = Expression - self._NoProcess = True self._Expr = ReplaceExprMacro(Expression.strip(), SymbolTable, @@ -293,13 +290,15 @@ class ValueExpression(object): self._Token = self._Expr if self.__IsNumberToken(): return self._Expr - + Token = '' try: Token = self._GetToken() - if type(Token) == type('') and Token.startswith('{') and Token.endswith('}') and self._Idx >= self._Len: - return self._Expr except BadExpression: pass + if type(Token) == type('') and Token.startswith('{') and Token.endswith('}') and self._Idx >= self._Len: + if len(Token) != len(self._Expr.replace(' ', '')): + raise BadExpression + return self._Expr self._Idx = 0 self._Token = '' @@ -454,13 +453,20 @@ class ValueExpression(object): Radix = 10 if self._Token.lower()[0:2] == '0x' and len(self._Token) > 2: Radix = 16 - if self._Token.startswith('"') or self._Token.startswith("'")\ - or self._Token.startswith("L'") or self._Token.startswith('L"'): + if self._Token.startswith('"') or self._Token.startswith('L"'): Flag = 0 for Index in range(len(self._Token)): - if self._Token[Index] in ['"', "'"]: + if self._Token[Index] in ['"']: Flag += 1 - if Flag == 2: + if Flag == 2 and self._Token.endswith('"'): + self._Token = ParseFieldValue(self._Token)[0] + return True + if self._Token.startswith("'") or self._Token.startswith("L'"): + Flag = 0 + for Index in range(len(self._Token)): + if self._Token[Index] in ["'"]: + Flag += 1 + if Flag == 2 and self._Token.endswith("'"): self._Token = ParseFieldValue(self._Token)[0] return True try: @@ -593,11 +599,10 @@ class ValueExpression(object): if self.HexPattern.match(self._LiteralToken): Token = self._LiteralToken[2:] - Token = Token.lstrip('0') if not Token: self._LiteralToken = '0x0' else: - self._LiteralToken = '0x' + Token.lower() + self._LiteralToken = '0x' + Token return True return False @@ -734,145 +739,159 @@ class ValueExpressionEx(ValueExpression): PcdValue = self.PcdValue try: PcdValue = ValueExpression.__call__(self, RealValue, Depth) + if self.PcdType == 'VOID*' and (PcdValue.startswith("'") or PcdValue.startswith("L'")): + raise BadExpression + elif self.PcdType in ['UINT8', 'UINT16', 'UINT32', 'UINT64', 'BOOLEAN'] and (PcdValue.startswith("'") or \ + PcdValue.startswith('"') or PcdValue.startswith("L'") or PcdValue.startswith('L"') or PcdValue.startswith('{')): + raise BadExpression except WrnExpression, Value: PcdValue = Value.result + except BadExpression: + if self.PcdType in ['UINT8', 'UINT16', 'UINT32', 'UINT64', 'BOOLEAN']: + PcdValue = PcdValue.strip() + if type(PcdValue) == type('') and PcdValue.startswith('{') and PcdValue.endswith('}'): + PcdValue = PcdValue[1:-1].split(',') + if type(PcdValue) == type([]): + TmpValue = 0 + Size = 0 + for Item in PcdValue: + if Item.startswith('UINT16'): + ItemSize = 2 + elif Item.startswith('UINT32'): + ItemSize = 4 + elif Item.startswith('UINT64'): + ItemSize = 8 + else: + ItemSize = 0 + Item = ValueExpressionEx(Item, self.PcdType, self._Symb)(True) + if ItemSize == 0: + ItemValue, ItemSize = ParseFieldValue(Item) + else: + ItemValue = ParseFieldValue(Item)[0] + + if type(ItemValue) == type(''): + ItemValue = int(ItemValue, 16) if ItemValue.startswith('0x') else int(ItemValue) + + TmpValue = (ItemValue << (Size * 8)) | TmpValue + Size = Size + ItemSize + else: + TmpValue, Size = ParseFieldValue(PcdValue) + if type(TmpValue) == type(''): + TmpValue = int(TmpValue) + else: + PcdValue = '0x%0{}X'.format(Size) % (TmpValue) + if TmpValue < 0: + raise BadExpression('Type %s PCD Value is negative' % self.PcdType) + if self.PcdType == 'UINT8' and Size > 1: + raise BadExpression('Type %s PCD Value Size is Larger than 1 byte' % self.PcdType) + if self.PcdType == 'UINT16' and Size > 2: + raise BadExpression('Type %s PCD Value Size is Larger than 2 byte' % self.PcdType) + if self.PcdType == 'UINT32' and Size > 4: + raise BadExpression('Type %s PCD Value Size is Larger than 4 byte' % self.PcdType) + if self.PcdType == 'UINT64' and Size > 8: + raise BadExpression('Type %s PCD Value Size is Larger than 8 byte' % self.PcdType) + if self.PcdType in ['VOID*']: + try: + TmpValue = long(PcdValue) + TmpList = [] + if TmpValue.bit_length() == 0: + PcdValue = '{0x00}' + else: + for I in range((TmpValue.bit_length() + 7) / 8): + TmpList.append('0x%02x' % ((TmpValue >> I * 8) & 0xff)) + PcdValue = '{' + ', '.join(TmpList) + '}' + except: + if PcdValue.strip().startswith('{'): + PcdValue = PcdValue.strip()[1:-1].strip() + Size = 0 + ValueStr = '' + TokenSpaceGuidName = '' + if PcdValue.startswith('GUID') and PcdValue.endswith(')'): + try: + TokenSpaceGuidName = re.search('GUID\((\w+)\)', PcdValue).group(1) + except: + pass + if TokenSpaceGuidName and TokenSpaceGuidName in self._Symb: + PcdValue = 'GUID(' + self._Symb[TokenSpaceGuidName] + ')' + elif TokenSpaceGuidName: + raise BadExpression('%s not found in DEC file' % TokenSpaceGuidName) + + ListItem, Size = ParseFieldValue(PcdValue) + elif PcdValue.startswith('DEVICE_PATH') and PcdValue.endswith(')'): + ListItem, Size = ParseFieldValue(PcdValue) + else: + ListItem = PcdValue.split(',') + + if type(ListItem) == type(0) or type(ListItem) == type(0L): + for Index in range(0, Size): + ValueStr += '0x%02X' % (int(ListItem) & 255) + ListItem >>= 8 + ValueStr += ', ' + PcdValue = '{' + ValueStr[:-2] + '}' + elif type(ListItem) == type(''): + if ListItem.startswith('{') and ListItem.endswith('}'): + PcdValue = ListItem + else: + LabelDict = {} + ReLabel = re.compile('LABEL\((\w+)\)') + ReOffset = re.compile('OFFSET_OF\((\w+)\)') + for Index, Item in enumerate(ListItem): + # for LABEL parse + Item = Item.strip() + try: + LabelList = ReLabel.findall(Item) + for Label in LabelList: + if Label not in LabelDict.keys(): + LabelDict[Label] = str(Index) + Item = ReLabel.sub('', Item) + except: + pass + try: + OffsetList = ReOffset.findall(Item) + except: + pass + for Offset in OffsetList: + if Offset in LabelDict.keys(): + Re = re.compile('OFFSET_OF\(%s\)'% Offset) + Item = Re.sub(LabelDict[Offset], Item) + else: + raise BadExpression('%s not defined before use' % Offset) + ValueType = "" + if Item.startswith('UINT16'): + ItemSize = 1 + ValueType = "UINT8" + elif Item.startswith('UINT16'): + ItemSize = 2 + ValueType = "UINT16" + elif Item.startswith('UINT32'): + ItemSize = 4 + elif Item.startswith('UINT64'): + ItemSize = 8 + else: + ItemSize = 0 + if ValueType: + TmpValue = ValueExpressionEx(Item, ValueType, self._Symb)(True) + else: + TmpValue = ValueExpressionEx(Item, self.PcdType, self._Symb)(True) + Item = '0x%x' % TmpValue if type(TmpValue) != type('') else TmpValue + if ItemSize == 0: + ItemValue, ItemSize = ParseFieldValue(Item) + else: + ItemValue = ParseFieldValue(Item)[0] + for I in range(0, ItemSize): + ValueStr += '0x%02X' % (int(ItemValue) & 255) + ItemValue >>= 8 + ValueStr += ', ' + Size += ItemSize + + if Size > 0: + PcdValue = '{' + ValueStr[:-2] + '}' if PcdValue == 'True': PcdValue = '1' if PcdValue == 'False': PcdValue = '0' - if self.PcdType in ['UINT8', 'UINT16', 'UINT32', 'UINT64', 'BOOLEAN']: - PcdValue = PcdValue.strip() - if type(PcdValue) == type('') and PcdValue.startswith('{') and PcdValue.endswith('}'): - PcdValue = PcdValue[1:-1].split(',') - if type(PcdValue) == type([]): - TmpValue = 0 - Size = 0 - for Item in PcdValue: - if Item.startswith('UINT16'): - ItemSize = 2 - elif Item.startswith('UINT32'): - ItemSize = 4 - elif Item.startswith('UINT64'): - ItemSize = 8 - else: - ItemSize = 0 - Item = ValueExpressionEx(Item, self.PcdType, self._Symb)(True) - if ItemSize == 0: - ItemValue, ItemSize = ParseFieldValue(Item) - else: - ItemValue = ParseFieldValue(Item)[0] - - if type(ItemValue) == type(''): - ItemValue = int(ItemValue, 16) if ItemValue.startswith('0x') else int(ItemValue) - - TmpValue = (ItemValue << (Size * 8)) | TmpValue - Size = Size + ItemSize - else: - TmpValue, Size = ParseFieldValue(PcdValue) - if type(TmpValue) == type(''): - TmpValue = int(TmpValue) - else: - PcdValue = '0x%0{}X'.format(Size) % (TmpValue) - if TmpValue < 0: - raise BadExpression('Type %s PCD Value is negative' % self.PcdType) - if self.PcdType == 'UINT8' and Size > 1: - raise BadExpression('Type %s PCD Value Size is Larger than 1 byte' % self.PcdType) - if self.PcdType == 'UINT16' and Size > 2: - raise BadExpression('Type %s PCD Value Size is Larger than 2 byte' % self.PcdType) - if self.PcdType == 'UINT32' and Size > 4: - raise BadExpression('Type %s PCD Value Size is Larger than 4 byte' % self.PcdType) - if self.PcdType == 'UINT64' and Size > 8: - raise BadExpression('Type %s PCD Value Size is Larger than 8 byte' % self.PcdType) - if self.PcdType in ['VOID*']: - try: - TmpValue = long(PcdValue) - TmpList = [] - if TmpValue.bit_length() == 0: - PcdValue = '{0x00}' - else: - for I in range((TmpValue.bit_length() + 7) / 8): - TmpList.append('0x%02x' % ((TmpValue >> I * 8) & 0xff)) - PcdValue = '{' + ', '.join(TmpList) + '}' - except: - if PcdValue.strip().startswith('{'): - PcdValue = PcdValue.strip()[1:-1].strip() - Size = 0 - ValueStr = '' - TokenSpaceGuidName = '' - if PcdValue.startswith('GUID') and PcdValue.endswith(')'): - try: - TokenSpaceGuidName = re.search('GUID\((\w+)\)', PcdValue).group(1) - except: - pass - if TokenSpaceGuidName and TokenSpaceGuidName in self._Symb: - PcdValue = 'GUID(' + self._Symb[TokenSpaceGuidName] + ')' - elif TokenSpaceGuidName: - raise BadExpression('%s not found in DEC file' % TokenSpaceGuidName) - - ListItem, Size = ParseFieldValue(PcdValue) - elif PcdValue.startswith('DEVICE_PATH') and PcdValue.endswith(')'): - ListItem, Size = ParseFieldValue(PcdValue) - else: - ListItem = PcdValue.split(',') - - if type(ListItem) == type(0) or type(ListItem) == type(0L): - for Index in range(0, Size): - ValueStr += '0x%02X' % (int(ListItem) & 255) - ListItem >>= 8 - ValueStr += ', ' - PcdValue = '{' + ValueStr[:-2] + '}' - elif type(ListItem) == type(''): - if ListItem.startswith('{') and ListItem.endswith('}'): - PcdValue = ListItem - else: - LabelDict = {} - ReLabel = re.compile('LABEL\((\w+)\)') - ReOffset = re.compile('OFFSET_OF\((\w+)\)') - for Index, Item in enumerate(ListItem): - # for LABEL parse - Item = Item.strip() - try: - LabelList = ReLabel.findall(Item) - for Label in LabelList: - if Label not in LabelDict.keys(): - LabelDict[Label] = str(Index) - Item = ReLabel.sub('', Item) - except: - pass - try: - OffsetList = ReOffset.findall(Item) - except: - pass - for Offset in OffsetList: - if Offset in LabelDict.keys(): - Re = re.compile('OFFSET_OF\(%s\)'% Offset) - Item = Re.sub(LabelDict[Offset], Item) - else: - raise BadExpression('%s not defined before use' % Offset) - if Item.startswith('UINT16'): - ItemSize = 2 - elif Item.startswith('UINT32'): - ItemSize = 4 - elif Item.startswith('UINT64'): - ItemSize = 8 - else: - ItemSize = 0 - TmpValue = ValueExpressionEx(Item, self.PcdType, self._Symb)(True) - Item = '0x%x' % TmpValue if type(TmpValue) != type('') else TmpValue - if ItemSize == 0: - ItemValue, ItemSize = ParseFieldValue(Item) - else: - ItemValue = ParseFieldValue(Item)[0] - for I in range(0, ItemSize): - ValueStr += '0x%02X' % (int(ItemValue) & 255) - ItemValue >>= 8 - ValueStr += ', ' - Size += ItemSize - - if Size > 0: - PcdValue = '{' + ValueStr[:-2] + '}' if RealValue: return PcdValue diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index f30d3f7e73..012e16a488 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -825,13 +825,14 @@ class DscBuildData(PlatformBuildClassObject): if ValueList[2] == '-1': EdkLogger.error('build', FORMAT_INVALID, "Pcd format incorrect.", File=self.MetaFile, Line=LineNo, ExtraData="%s.%s|%s" % (TokenSpaceGuid, PcdCName, Setting)) - if ValueList[Index] and PcdType not in [MODEL_PCD_FEATURE_FLAG, MODEL_PCD_FIXED_AT_BUILD]: + if ValueList[Index]: + DatumType = self._DecPcds[PcdCName, TokenSpaceGuid].DatumType try: - ValueList[Index] = ValueExpression(ValueList[Index], GlobalData.gPlatformPcds)(True) - except WrnExpression, Value: - ValueList[Index] = Value.result + ValueList[Index] = ValueExpressionEx(ValueList[Index], DatumType, self._GuidDict)(True) except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, Value, File=self.MetaFile, Line=self._LineIndex + 1) + EdkLogger.error('Parser', FORMAT_INVALID, Value, File=self.MetaFile, Line=LineNo, + ExtraData="PCD [%s.%s] Value \"%s\" " % ( + TokenSpaceGuid, PcdCName, ValueList[Index])) except EvaluationException, Excpt: if hasattr(Excpt, 'Pcd'): if Excpt.Pcd in GlobalData.gPlatformOtherPcds: @@ -845,13 +846,8 @@ class DscBuildData(PlatformBuildClassObject): else: EdkLogger.error('Parser', FORMAT_INVALID, "Invalid expression: %s" % str(Excpt), File=self.MetaFile, Line=LineNo) + if ValueList[Index]: - DatumType = self._DecPcds[PcdCName, TokenSpaceGuid].DatumType - try: - ValueList[Index] = ValueExpressionEx(ValueList[Index], DatumType, self._GuidDict)(True) - except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, Value, File=self.MetaFile, Line=LineNo, - ExtraData="PCD [%s.%s] Value \"%s\" " % (TokenSpaceGuid, PcdCName, ValueList[Index])) Valid, ErrStr = CheckPcdDatum(self._DecPcds[PcdCName, TokenSpaceGuid].DatumType, ValueList[Index]) if not Valid: EdkLogger.error('build', FORMAT_INVALID, ErrStr, File=self.MetaFile, Line=LineNo, @@ -860,6 +856,9 @@ class DscBuildData(PlatformBuildClassObject): if self._DecPcds[PcdCName, TokenSpaceGuid].DatumType.strip() != ValueList[1].strip(): EdkLogger.error('build', FORMAT_INVALID, ErrStr , File=self.MetaFile, Line=LineNo, ExtraData="%s.%s|%s" % (TokenSpaceGuid, PcdCName, Setting)) + if (TokenSpaceGuid + '.' + PcdCName) in GlobalData.gPlatformPcds: + if GlobalData.gPlatformPcds[TokenSpaceGuid + '.' + PcdCName] != ValueList[Index]: + GlobalData.gPlatformPcds[TokenSpaceGuid + '.' + PcdCName] = ValueList[Index] return ValueList def _FilterPcdBySkuUsage(self,Pcds): diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py index 8f4b5e5cc1..c928cef70f 100644 --- a/BaseTools/Source/Python/Workspace/MetaFileParser.py +++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py @@ -1593,6 +1593,8 @@ class DscParser(MetaFileParser): ValList[Index] = ValueExpression(PcdValue, self._Macros)(True) except WrnExpression, Value: ValList[Index] = Value.result + except: + pass if ValList[Index] == 'True': ValList[Index] = '1' @@ -1989,14 +1991,6 @@ class DecParser(MetaFileParser): PcdValue = ValueList[0] if PcdValue: - try: - ValueList[0] = ValueExpression(PcdValue, self._AllPcdDict)(True) - except WrnExpression, Value: - ValueList[0] = Value.result - except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, Value, File=self.MetaFile, Line=self._LineIndex + 1) - - if ValueList[0]: try: ValueList[0] = ValueExpressionEx(ValueList[0], ValueList[1], self._GuidDict)(True) except BadExpression, Value: diff --git a/BaseTools/Source/Python/build/BuildReport.py b/BaseTools/Source/Python/build/BuildReport.py index f2a6e6d87e..53d0039c51 100644 --- a/BaseTools/Source/Python/build/BuildReport.py +++ b/BaseTools/Source/Python/build/BuildReport.py @@ -37,6 +37,7 @@ from Common.InfClassObject import gComponentType2ModuleType from Common.BuildToolError import FILE_WRITE_FAILURE from Common.BuildToolError import CODE_ERROR from Common.BuildToolError import COMMAND_FAILURE +from Common.BuildToolError import FORMAT_INVALID from Common.LongFilePathSupport import OpenLongFilePath as open from Common.MultipleWorkspace import MultipleWorkspace as mws import Common.GlobalData as GlobalData @@ -45,7 +46,7 @@ from Common.Misc import PathClass from Common.String import NormPath from Common.DataType import * import collections -from Common.Expression import ValueExpressionEx +from Common.Expression import * ## Pattern to extract contents in EDK DXS files gDxsDependencyPattern = re.compile(r"DEPENDENCY_START(.+)DEPENDENCY_END", re.DOTALL) @@ -955,7 +956,11 @@ class PcdReport(object): DscDefaultValBak = DscDefaultValue DscDefaultValue = self.FdfPcdSet.get((Pcd.TokenCName, Key), DscDefaultValue) if DscDefaultValue != DscDefaultValBak: - DscDefaultValue = ValueExpressionEx(DscDefaultValue, Pcd.DatumType, self._GuidDict)(True) + try: + DscDefaultValue = ValueExpressionEx(DscDefaultValue, Pcd.DatumType, self._GuidDict)(True) + except BadExpression, Value: + EdkLogger.error('BuildReport', FORMAT_INVALID, "PCD Value: %s, Type: %s" %(DscDefaultValue, Pcd.DatumType)) + InfDefaultValue = None PcdValue = DecDefaultValue