diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index 2b3f93c3bc..46e94d47e4 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -295,7 +295,7 @@ class WorkspaceAutoGen(AutoGen): SkippedArchList = set(self.ArchList).symmetric_difference(set(self.Platform.SupArchList)) EdkLogger.verbose("\nArch [%s] is ignored because the platform supports [%s] only!" % (" ".join(SkippedArchList), " ".join(self.Platform.SupArchList))) - self.ArchList = tuple(ArchList) + self.ArchList = tuple(sorted(ArchList)) # Validate build target if self.BuildTarget not in self.Platform.BuildTargets: @@ -616,17 +616,17 @@ class WorkspaceAutoGen(AutoGen): # content = 'gCommandLineDefines: ' content += str(GlobalData.gCommandLineDefines) - content += os.linesep + content += "\n" content += 'BuildOptionPcd: ' content += str(GlobalData.BuildOptionPcd) - content += os.linesep + content += "\n" content += 'Active Platform: ' content += str(self.Platform) - content += os.linesep + content += "\n" if self.FdfFile: content += 'Flash Image Definition: ' content += str(self.FdfFile) - content += os.linesep + content += "\n" SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False) # @@ -636,7 +636,7 @@ class WorkspaceAutoGen(AutoGen): if Pa.PcdTokenNumber: if Pa.DynamicPcdList: for Pcd in Pa.DynamicPcdList: - PcdTokenNumber += os.linesep + PcdTokenNumber += "\n" PcdTokenNumber += str((Pcd.TokenCName, Pcd.TokenSpaceGuidCName)) PcdTokenNumber += ' : ' PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName]) @@ -677,7 +677,7 @@ class WorkspaceAutoGen(AutoGen): if not os.path.exists(self.BuildDir): os.makedirs(self.BuildDir) with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file: - for f in AllWorkSpaceMetaFiles: + for f in sorted(AllWorkSpaceMetaFiles): print(f, file=file) return True @@ -1598,6 +1598,9 @@ class PlatformAutoGen(AutoGen): self._DynamicPcdList.extend(list(UnicodePcdArray)) self._DynamicPcdList.extend(list(HiiPcdArray)) self._DynamicPcdList.extend(list(OtherPcdArray)) + #python3.6 set is not ordered at all + self._DynamicPcdList = sorted(self._DynamicPcdList, key=lambda x:(x.TokenSpaceGuidCName, x.TokenCName)) + self._NonDynamicPcdList = sorted(self._NonDynamicPcdList, key=lambda x: (x.TokenSpaceGuidCName, x.TokenCName)) allskuset = [(SkuName, Sku.SkuId) for pcd in self._DynamicPcdList for (SkuName, Sku) in pcd.SkuInfoList.items()] for pcd in self._DynamicPcdList: if len(pcd.SkuInfoList) == 1: @@ -2374,7 +2377,7 @@ class PlatformAutoGen(AutoGen): list(PlatformModuleOptions.keys()) + list(ModuleTypeOptions.keys()) + list(self.ToolDefinition.keys())) BuildOptions = defaultdict(lambda: defaultdict(str)) - for Tool in AllTools: + for Tool in sorted(AllTools): for Options in [self.ToolDefinition, ModuleOptions, PlatformOptions, ModuleTypeOptions, PlatformModuleOptions]: if Tool not in Options: continue @@ -3156,12 +3159,12 @@ class ModuleAutoGen(AutoGen): @cached_property def IntroTargetList(self): self.Targets - return self._IntroBuildTargetList + return sorted(self._IntroBuildTargetList, key=lambda x: str(x.Target)) @cached_property def CodaTargetList(self): self.Targets - return self._FinalBuildTargetList + return sorted(self._FinalBuildTargetList, key=lambda x: str(x.Target)) @cached_property def FileTypes(self): @@ -3889,7 +3892,7 @@ class ModuleAutoGen(AutoGen): if os.path.exists (self.TimeStampPath): os.remove (self.TimeStampPath) with open(self.TimeStampPath, 'w+') as file: - for f in FileSet: + for f in sorted(FileSet): print(f, file=file) # Ignore generating makefile when it is a binary module diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py index a21880f317..4db5330e51 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -1011,7 +1011,10 @@ def CreateModulePcdCode(Info, AutoGenC, AutoGenH, Pcd): Value = Value[:-1] ValueNumber = int (Value, 0) except: - EdkLogger.error("build", AUTOGEN_ERROR, + try: + ValueNumber = int(Value.lstrip('0')) + except: + EdkLogger.error("build", AUTOGEN_ERROR, "PCD value is not valid dec or hex number for datum type [%s] of PCD %s.%s" % (Pcd.DatumType, Pcd.TokenSpaceGuidCName, TokenCName), ExtraData="[%s]" % str(Info)) if ValueNumber < 0: diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index 0051aaf41f..2f6fe06764 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -917,7 +917,7 @@ cleanlib: # # Extract common files list in the dependency files # - for File in DepSet: + for File in sorted(DepSet, key=lambda x: str(x)): self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros)) for File in FileDependencyDict: @@ -926,11 +926,11 @@ cleanlib: continue NewDepSet = set(FileDependencyDict[File]) NewDepSet -= DepSet - FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet) + FileDependencyDict[File] = ["$(COMMON_DEPS)"] + sorted(NewDepSet, key=lambda x: str(x)) # Convert target description object to target string in makefile for Type in self._AutoGenObject.Targets: - for T in self._AutoGenObject.Targets[Type]: + for T in sorted(self._AutoGenObject.Targets[Type], key=lambda x: str(x)): # Generate related macros if needed if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: self.FileListMacros[T.FileListMacro] = [] @@ -1097,7 +1097,7 @@ cleanlib: DependencySet.update(ForceList) if File in DependencySet: DependencySet.remove(File) - DependencyList = list(DependencySet) # remove duplicate ones + DependencyList = sorted(DependencySet, key=lambda x: str(x)) # remove duplicate ones return DependencyList diff --git a/BaseTools/Source/Python/AutoGen/GenPcdDb.py b/BaseTools/Source/Python/AutoGen/GenPcdDb.py index a546ecbbb7..68f92ef063 100644 --- a/BaseTools/Source/Python/AutoGen/GenPcdDb.py +++ b/BaseTools/Source/Python/AutoGen/GenPcdDb.py @@ -1348,7 +1348,7 @@ def CreatePcdDatabasePhaseSpecificAutoGen (Platform, DynamicPcdList, Phase): DbValueList.append(Sku.DefaultValue) - Pcd.TokenTypeList = list(set(Pcd.TokenTypeList)) + Pcd.TokenTypeList = sorted(set(Pcd.TokenTypeList)) if Pcd.DatumType == TAB_VOID: Dict['SIZE_TABLE_CNAME'].append(CName) Dict['SIZE_TABLE_GUID'].append(TokenSpaceGuid) @@ -1449,7 +1449,7 @@ def CreatePcdDatabasePhaseSpecificAutoGen (Platform, DynamicPcdList, Phase): Dict['PCD_CNAME_LENGTH'][GeneratedTokenNumber] = len(CNameBinArray.split(",")) - Pcd.TokenTypeList = list(set(Pcd.TokenTypeList)) + Pcd.TokenTypeList = sorted(set(Pcd.TokenTypeList)) # search the Offset and Table, used by LocalTokenNumberTableOffset if 'PCD_TYPE_HII' in Pcd.TokenTypeList: diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py index 71ab6da529..370b4bbb49 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -1597,15 +1597,19 @@ def CheckPcdDatum(Type, Value): return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\ ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type) elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]: - if Value and int(Value, 0) < 0: - return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type) try: - Value = int(Value, 0) - if Value > MAX_VAL_TYPE[Type]: - return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type) + Val = int(Value, 0) except: - return False, "Invalid value [%s] of type [%s];"\ - " must be a hexadecimal, decimal or octal in C language format." % (Value, Type) + try: + Val = int(Value.lstrip('0')) + except: + return False, "Invalid value [%s] of type [%s];" \ + " must be a hexadecimal, decimal or octal in C language format." % (Value, Type) + if Val > MAX_VAL_TYPE[Type]: + return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type) + if Val < 0: + return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type) + else: return True, "StructurePcd" diff --git a/BaseTools/Source/Python/Common/StringUtils.py b/BaseTools/Source/Python/Common/StringUtils.py index fe899b11d8..794f4573fe 100644 --- a/BaseTools/Source/Python/Common/StringUtils.py +++ b/BaseTools/Source/Python/Common/StringUtils.py @@ -612,7 +612,7 @@ def PreCheck(FileName, FileContent, SupSectionTag): # # Regenerate FileContent # - NewFileContent = NewFileContent + Line + '\r\n' + NewFileContent = NewFileContent + Line + '\n' if IsFailed: EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) diff --git a/BaseTools/Source/Python/GenFds/FvImageSection.py b/BaseTools/Source/Python/GenFds/FvImageSection.py index 5cc5b969fd..f15df23514 100644 --- a/BaseTools/Source/Python/GenFds/FvImageSection.py +++ b/BaseTools/Source/Python/GenFds/FvImageSection.py @@ -102,6 +102,8 @@ class FvImageSection(FvImageSectionClassObject): Fv = GenFdsGlobalVariable.FdfParser.Profile.FvDict.get(self.FvName) if Fv is not None: self.Fv = Fv + if not self.FvAddr and self.Fv.BaseAddress: + self.FvAddr = self.Fv.BaseAddress FvFileName = Fv.AddToBuffer(Buffer, self.FvAddr, MacroDict = Dict, Flag=IsMakefile) if Fv.FvAlignment is not None: if self.Alignment is None: diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py index ed1bd33fdb..32824ae80e 100644 --- a/BaseTools/Source/Python/GenFds/GenFds.py +++ b/BaseTools/Source/Python/GenFds/GenFds.py @@ -321,6 +321,8 @@ def main(): continue for RegionData in RegionObj.RegionDataList: if FvObj.UiFvName.upper() == RegionData.upper(): + if not FvObj.BaseAddress: + FvObj.BaseAddress = '0x%x' % (int(FdObj.BaseAddress, 0) + RegionObj.Offset) if FvObj.FvRegionInFD: if FvObj.FvRegionInFD != RegionObj.Size: EdkLogger.error("GenFds", FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different value." %FvObj.UiFvName) @@ -607,16 +609,21 @@ class GenFds : ModuleList = [] FileGuidList = [] GuidPattern = gGuidPattern + VariableGuidSet = set() for Arch in ArchList: PlatformDataBase = BuildDb.BuildObject[GenFdsGlobalVariable.ActivePlatform, Arch, GenFdsGlobalVariable.TargetName, GenFdsGlobalVariable.ToolChainTag] PkgList = GenFdsGlobalVariable.WorkSpace.GetPackageList(GenFdsGlobalVariable.ActivePlatform, Arch, GenFdsGlobalVariable.TargetName, GenFdsGlobalVariable.ToolChainTag) for P in PkgList: PkgGuidDict.update(P.Guids) - for Name, Guid in PlatformDataBase.Pcds: + for Name, Guid in sorted(PlatformDataBase.Pcds): Pcd = PlatformDataBase.Pcds[Name, Guid] if Pcd.Type in [TAB_PCDS_DYNAMIC_HII, TAB_PCDS_DYNAMIC_EX_HII]: for SkuId in Pcd.SkuInfoList: Sku = Pcd.SkuInfoList[SkuId] + if Sku.VariableGuid in VariableGuidSet: + continue + else: + VariableGuidSet.add(Sku.VariableGuid) if Sku.VariableGuid and Sku.VariableGuid in PkgGuidDict.keys(): GuidDict[Sku.VariableGuid] = PkgGuidDict[Sku.VariableGuid] for ModuleFile in PlatformDataBase.Modules: diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index c717401cab..02aae3a67b 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -128,7 +128,7 @@ def GetDependencyList(FileStack, SearchPathList): if len(FileContent) == 0: continue - + IncludedFileList = [] if FileContent[0] == 0xff or FileContent[0] == 0xfe: FileContent = str(FileContent, "utf-16") IncludedFileList = gIncludePattern.findall(FileContent) diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py index 50ffe308aa..fdf7738a31 100644 --- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py +++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py @@ -164,7 +164,7 @@ class WorkspaceDatabase(object): os.remove(DbPath) # create db with optimized parameters - self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED') + self.Conn = sqlite3.connect(DbPath, isolation_level=None) self.Conn.execute("PRAGMA synchronous=OFF") self.Conn.execute("PRAGMA temp_store=MEMORY") self.Conn.execute("PRAGMA count_changes=OFF") diff --git a/BaseTools/Source/Python/build/BuildReport.py b/BaseTools/Source/Python/build/BuildReport.py index fd9294287f..06cf419931 100644 --- a/BaseTools/Source/Python/build/BuildReport.py +++ b/BaseTools/Source/Python/build/BuildReport.py @@ -79,7 +79,7 @@ gGlueLibEntryPoint = re.compile(r"__EDKII_GLUE_MODULE_ENTRY_POINT__\s*=\s*(\w+)" gLineMaxLength = 120 ## Tags for end of line in report -gEndOfLine = "\r\n" +gEndOfLine = "\n" ## Tags for section start, end and separator gSectionStart = ">" + "=" * (gLineMaxLength - 2) + "<" @@ -1031,7 +1031,10 @@ class PcdReport(object): if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES: - PcdValueNumber = int(PcdValue.strip(), 0) + try: + PcdValueNumber = int(PcdValue.strip(), 0) + except: + PcdValueNumber = int(PcdValue.lstrip('0')) if DecDefaultValue is None: DecMatch = True else: @@ -1047,7 +1050,10 @@ class PcdReport(object): if DscDefaultValue is None: DscMatch = True else: - DscDefaultValueNumber = int(DscDefaultValue.strip(), 0) + try: + DscDefaultValueNumber = int(DscDefaultValue.strip(), 0) + except: + DscDefaultValueNumber = int(DscDefaultValue.lstrip('0')) DscMatch = (DscDefaultValueNumber == PcdValueNumber) else: if DecDefaultValue is None: @@ -1152,7 +1158,10 @@ class PcdReport(object): for ModulePath in ModuleOverride: ModuleDefault = ModuleOverride[ModulePath] if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES: - ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0) + try: + ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0) + except: + ModulePcdDefaultValueNumber = int(ModuleDefault.lstrip('0')) Match = (ModulePcdDefaultValueNumber == PcdValueNumber) if Pcd.DatumType == 'BOOLEAN': ModuleDefault = str(ModulePcdDefaultValueNumber) @@ -1231,7 +1240,10 @@ class PcdReport(object): if Value.startswith(('0x', '0X')): Value = '{} ({:d})'.format(Value, int(Value, 0)) else: - Value = "0x{:X} ({})".format(int(Value, 0), Value) + try: + Value = "0x{:X} ({})".format(int(Value, 0), Value) + except: + Value = "0x{:X} ({})".format(int(Value.lstrip('0')), Value) FileWrite(File, ' %*s = %s' % (self.MaxLen + 19, 'DEC DEFAULT', Value)) if IsStructure: self.PrintStructureInfo(File, Pcd.DefaultValues)