BaseTools/UPT: Support Multiple Installation

Add a new feature to UPT to support installing
multiple DIST packages in one time.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Hess Chen <hesheng.chen@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>

BaseTools/Source/Python/UPT/Core/DependencyRules.py

@@ -44,12 +44,24 @@ DEPEX_CHECK_PACKAGE_NOT_FOUND, DEPEX_CHECK_DP_NOT_FOUND) = (0, 1, 2, 3)
 # @param object:      Inherited from object class
 #
 class DependencyRules(object):
-    def __init__(self, Datab):
+    def __init__(self, Datab, ToBeInstalledPkgList=None):
         self.IpiDb = Datab
         self.WsPkgList = GetWorkspacePackage()
         self.WsModuleList = GetWorkspaceModule()
-        self.PkgsToBeDepend = []
+
+        self.PkgsToBeDepend = [(PkgInfo[1], PkgInfo[2]) for PkgInfo in self.WsPkgList]
+
+        # Add package info from the DIST to be installed.
+        self.PkgsToBeDepend.extend(self.GenToBeInstalledPkgList(ToBeInstalledPkgList))
         
+    def GenToBeInstalledPkgList(self, ToBeInstalledPkgList):
+        RtnList = []
+        for Dist in ToBeInstalledPkgList:
+            for Package in Dist.PackageSurfaceArea:
+                RtnList.append((Package[0], Package[1]))
+
+        return RtnList
+
     ## Check whether a module exists by checking the Guid+Version+Name+Path combination
     #
     # @param Guid:  Guid of a module
@@ -182,7 +194,6 @@ class DependencyRules(object):
     #          False else
     #
     def CheckInstallDpDepexSatisfied(self, DpObj):
-        self.PkgsToBeDepend = [(PkgInfo[1], PkgInfo[2]) for PkgInfo in self.WsPkgList]
         return self.CheckDpDepexSatisfied(DpObj)
 
     # # Check whether multiple DP depex satisfied by current workspace for Install
@@ -192,7 +203,6 @@ class DependencyRules(object):
     #          False else
     #
     def CheckTestInstallPdDepexSatisfied(self, DpObjList):
-        self.PkgsToBeDepend = [(PkgInfo[1], PkgInfo[2]) for PkgInfo in self.WsPkgList]
         for DpObj in DpObjList:
             if self.CheckDpDepexSatisfied(DpObj):
                 for PkgKey in DpObj.PackageSurfaceArea.keys():

BaseTools/Source/Python/UPT/GenMetaFile/GenMetaFileMisc.py

@@ -79,6 +79,10 @@ def AddExternToDefineSec(SectionDict, Arch, ExternList):
 # Using TokenSpaceGuidValue and Token to obtain PcdName from DEC file
 #
 def ObtainPcdName(Packages, TokenSpaceGuidValue, Token):
+    TokenSpaceGuidName = ''
+    PcdCName = ''
+    TokenSpaceGuidNameFound = False
+
     for PackageDependency in Packages:
         #
         # Generate generic comment
@@ -86,6 +90,7 @@ def ObtainPcdName(Packages, TokenSpaceGuidValue, Token):
         Guid = PackageDependency.GetGuid()
         Version = PackageDependency.GetVersion()
 
+        Path = None
         #
         # find package path/name
         # 
@@ -95,41 +100,58 @@ def ObtainPcdName(Packages, TokenSpaceGuidValue, Token):
                     Path = PkgInfo[3]
                     break
 
-        DecFile = None
+        # The dependency package in workspace
-        if Path not in GlobalData.gPackageDict:
+        if Path:
-            DecFile = Dec(Path)
+            DecFile = None
-            GlobalData.gPackageDict[Path] = DecFile
+            if Path not in GlobalData.gPackageDict:
-        else:
+                DecFile = Dec(Path)
-            DecFile = GlobalData.gPackageDict[Path]
+                GlobalData.gPackageDict[Path] = DecFile
+            else:
+                DecFile = GlobalData.gPackageDict[Path]
 
-        DecGuidsDict = DecFile.GetGuidSectionObject().ValueDict
+            DecGuidsDict = DecFile.GetGuidSectionObject().ValueDict
-        DecPcdsDict = DecFile.GetPcdSectionObject().ValueDict
+            DecPcdsDict = DecFile.GetPcdSectionObject().ValueDict
 
-        TokenSpaceGuidName = ''
+            TokenSpaceGuidName = ''
-        PcdCName = ''
+            PcdCName = ''
-        TokenSpaceGuidNameFound = False
+            TokenSpaceGuidNameFound = False
 
-        #
+            #
-        # Get TokenSpaceGuidCName from Guids section 
+            # Get TokenSpaceGuidCName from Guids section
-        #
+            #
-        for GuidKey in DecGuidsDict:
+            for GuidKey in DecGuidsDict:
-            GuidList = DecGuidsDict[GuidKey]
+                GuidList = DecGuidsDict[GuidKey]
-            for GuidItem in GuidList:
+                for GuidItem in GuidList:
-                if TokenSpaceGuidValue.upper() == GuidItem.GuidString.upper():
+                    if TokenSpaceGuidValue.upper() == GuidItem.GuidString.upper():
-                    TokenSpaceGuidName = GuidItem.GuidCName
+                        TokenSpaceGuidName = GuidItem.GuidCName
-                    TokenSpaceGuidNameFound = True
+                        TokenSpaceGuidNameFound = True
+                        break
+                if TokenSpaceGuidNameFound:
                     break
-            if TokenSpaceGuidNameFound:
+            #
-                break
+            # Retrieve PcdCName from Pcds Section
-        #
+            #
-        # Retrieve PcdCName from Pcds Section
+            for PcdKey in DecPcdsDict:
-        #
+                PcdList = DecPcdsDict[PcdKey]
-        for PcdKey in DecPcdsDict:
+                for PcdItem in PcdList:
-            PcdList = DecPcdsDict[PcdKey]
+                    if TokenSpaceGuidName == PcdItem.TokenSpaceGuidCName and Token == PcdItem.TokenValue:
-            for PcdItem in PcdList:
+                        PcdCName = PcdItem.TokenCName
-                if TokenSpaceGuidName == PcdItem.TokenSpaceGuidCName and Token == PcdItem.TokenValue:
+                        return TokenSpaceGuidName, PcdCName
-                    PcdCName = PcdItem.TokenCName
+
-                    return TokenSpaceGuidName, PcdCName
+        # The dependency package in ToBeInstalledDist
+        else:
+            for Dist in GlobalData.gTO_BE_INSTALLED_DIST_LIST:
+                for Package in Dist.PackageSurfaceArea.values():
+                    if Guid == Package.Guid:
+                        for GuidItem in Package.GuidList:
+                            if TokenSpaceGuidValue.upper() == GuidItem.Guid.upper():
+                                TokenSpaceGuidName = GuidItem.CName
+                                TokenSpaceGuidNameFound = True
+                                break
+                        for PcdItem in Package.PcdList:
+                            if TokenSpaceGuidName == PcdItem.TokenSpaceGuidCName and Token == PcdItem.Token:
+                                PcdCName = PcdItem.CName
+                                return TokenSpaceGuidName, PcdCName
 
     return TokenSpaceGuidName, PcdCName
 

BaseTools/Source/Python/UPT/InstallPkg.py

@@ -1,7 +1,7 @@
 ## @file
 # Install distribution package.
 #
-# Copyright (c) 2011 - 2015, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2011 - 2017, Intel Corporation. All rights reserved.<BR>
 #
 # This program and the accompanying materials are licensed and made available 
 # under the terms and conditions of the BSD License which accompanies this 
@@ -133,16 +133,16 @@ def InstallNewFile(WorkspaceDir, File):
 #
 # UnZipDp
 #
-def UnZipDp(WorkspaceDir, DpPkgFileName):
+def UnZipDp(WorkspaceDir, DpPkgFileName, Index=1):
     ContentZipFile = None
     Logger.Quiet(ST.MSG_UZIP_PARSE_XML)
     DistFile = PackageFile(DpPkgFileName)
     
     DpDescFileName, ContentFileName = GetDPFile(DistFile.GetZipFile())
     
-    GlobalData.gUNPACK_DIR = os.path.normpath(os.path.join(WorkspaceDir, ".tmp"))
+    TempDir = os.path.normpath(os.path.join(WorkspaceDir, "Conf/.tmp%s" % str(Index)))
-    DistPkgFile = DistFile.UnpackFile(DpDescFileName,
+    GlobalData.gUNPACK_DIR.append(TempDir)
-        os.path.normpath(os.path.join(GlobalData.gUNPACK_DIR, DpDescFileName)))
+    DistPkgFile = DistFile.UnpackFile(DpDescFileName, os.path.normpath(os.path.join(TempDir, DpDescFileName)))
     if not DistPkgFile:
         Logger.Error("InstallPkg", FILE_NOT_FOUND, ST.ERR_FILE_BROKEN %DpDescFileName)
     
@@ -159,23 +159,15 @@ def UnZipDp(WorkspaceDir, DpPkgFileName):
     #
     # unzip contents.zip file
     #
-    ContentFile = DistFile.UnpackFile(ContentFileName,
+    ContentFile = DistFile.UnpackFile(ContentFileName, os.path.normpath(os.path.join(TempDir, ContentFileName)))
-        os.path.normpath(os.path.join(GlobalData.gUNPACK_DIR, ContentFileName)))
     if not ContentFile:
         Logger.Error("InstallPkg", FILE_NOT_FOUND,
             ST.ERR_FILE_BROKEN % ContentFileName)
 
-    FilePointer = __FileHookOpen__(ContentFile, "rb")
-    #
-    # Assume no archive comment.
-    #
-    FilePointer.seek(0, SEEK_SET)
-    FilePointer.seek(0, SEEK_END)
     #
     # Get file size
     #                
-    FileSize = FilePointer.tell()
+    FileSize = os.path.getsize(ContentFile)
-    FilePointer.close()
                
     if FileSize != 0:        
         ContentZipFile = PackageFile(ContentFile)
@@ -202,8 +194,8 @@ def GetPackageList(DistPkg, Dep, WorkspaceDir, Options, ContentZipFile, ModuleLi
         PackagePath = Path
         Package = DistPkg.PackageSurfaceArea[Guid, Version, Path]
         Logger.Info(ST.MSG_INSTALL_PACKAGE % Package.GetName())
-        if Dep.CheckPackageExists(Guid, Version):
+#         if Dep.CheckPackageExists(Guid, Version):
-            Logger.Info(ST.WRN_PACKAGE_EXISTED %(Guid, Version))
+#             Logger.Info(ST.WRN_PACKAGE_EXISTED %(Guid, Version))
         if Options.UseGuidedPkgPath:
             GuidedPkgPath = "%s_%s_%s" % (Package.GetName(), Guid, Version)
             NewPackagePath = InstallNewPackage(WorkspaceDir, GuidedPkgPath, Options.CustomPath)
@@ -509,29 +501,40 @@ def GenToolMisc(DistPkg, WorkspaceDir, ContentZipFile):
 # @param  Options: command Options
 #
 def Main(Options = None):
-    ContentZipFile, DistFile = None, None
-
     try:
         DataBase = GlobalData.gDB
         WorkspaceDir = GlobalData.gWORKSPACE
         if not Options.PackageFile:
             Logger.Error("InstallPkg", OPTION_MISSING, ExtraData=ST.ERR_SPECIFY_PACKAGE)
         
-        #
+        # Get all Dist Info
-        # unzip dist.pkg file
+        DistInfoList = []
-        #
+        DistPkgList = []
-        DistPkg, ContentZipFile, DpPkgFileName, DistFile = UnZipDp(WorkspaceDir, Options.PackageFile)
+        Index = 1
+        for ToBeInstalledDist in Options.PackageFile:
+            #
+            # unzip dist.pkg file
+            #
+            DistInfoList.append(UnZipDp(WorkspaceDir, ToBeInstalledDist, Index))
+            DistPkgList.append(DistInfoList[-1][0])
+            Index += 1
 
-        #
+            #
-        # check dependency
+            # Add dist
-        #
+            #
-        Dep = DependencyRules(DataBase)
+            GlobalData.gTO_BE_INSTALLED_DIST_LIST.append(DistInfoList[-1][0])
-        CheckInstallDpx(Dep, DistPkg)
 
-        #
+        # Check for dependency
-        # Install distribution
+        Dep = DependencyRules(DataBase, DistPkgList)
-        #
+
-        InstallDp(DistPkg, DpPkgFileName, ContentZipFile, Options, Dep, WorkspaceDir, DataBase)
+        for ToBeInstalledDist in DistInfoList:
+            CheckInstallDpx(Dep, ToBeInstalledDist[0], ToBeInstalledDist[2])
+
+            #
+            # Install distribution
+            #
+            InstallDp(ToBeInstalledDist[0], ToBeInstalledDist[2], ToBeInstalledDist[1],
+                      Options, Dep, WorkspaceDir, DataBase)
         ReturnCode = 0
         
     except FatalError, XExcept:
@@ -556,16 +559,16 @@ def Main(Options = None):
         Logger.Quiet(ST.MSG_PYTHON_ON % (python_version(),
             platform) + format_exc())
     finally:
-        if ReturnCode != UPT_ALREADY_INSTALLED_ERROR:
+        Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_STARTED)
-            Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_STARTED)
+        for ToBeInstalledDist in DistInfoList:
-            if DistFile:
+            if ToBeInstalledDist[3]:
-                DistFile.Close()
+                ToBeInstalledDist[3].Close()
-            if ContentZipFile:
+            if ToBeInstalledDist[1]:
-                ContentZipFile.Close()
+                ToBeInstalledDist[1].Close()
-            if GlobalData.gUNPACK_DIR:
+        for TempDir in GlobalData.gUNPACK_DIR:
-                rmtree(GlobalData.gUNPACK_DIR)
+            rmtree(TempDir)
-                GlobalData.gUNPACK_DIR = None
+        GlobalData.gUNPACK_DIR = []
-            Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_DONE)
+        Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_DONE)
     if ReturnCode == 0:
         Logger.Quiet(ST.MSG_FINISH)
     return ReturnCode
@@ -609,14 +612,15 @@ def BackupDist(DpPkgFileName, Guid, Version, WorkspaceDir):
 #   @param  Dep: the DependencyRules instance that used to check dependency
 #   @param  DistPkg: the distribution object
 #
-def CheckInstallDpx(Dep, DistPkg):
+def CheckInstallDpx(Dep, DistPkg, DistPkgFileName):
     #
     # Check distribution package installed or not
     #
     if Dep.CheckDpExists(DistPkg.Header.GetGuid(),
         DistPkg.Header.GetVersion()):
-        Logger.Error("InstallPkg", UPT_ALREADY_INSTALLED_ERROR,
+        Logger.Error("InstallPkg",
-            ST.WRN_DIST_PKG_INSTALLED)
+                     UPT_ALREADY_INSTALLED_ERROR,
+                     ST.WRN_DIST_PKG_INSTALLED % os.path.basename(DistPkgFileName))
     #
     # Check distribution dependency (all module dependency should be
     # satisfied)

BaseTools/Source/Python/UPT/Library/GlobalData.py

@@ -1,7 +1,7 @@
 ## @file
 # This file is used to define common static strings and global data used by UPT
 #
-# Copyright (c) 2011 - 2014, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2011 - 2017, Intel Corporation. All rights reserved.<BR>
 #
 # This program and the accompanying materials are licensed and made available 
 # under the terms and conditions of the BSD License which accompanies this 
@@ -87,7 +87,7 @@ gWARNING_AS_ERROR = False
 #
 # Used to specify the temp directory to hold the unpacked distribution files
 #
-gUNPACK_DIR = None
+gUNPACK_DIR = []
 
 #
 # Flag used to mark whether the INF file is Binary INF or not.
@@ -109,3 +109,8 @@ gPackageDict = {}
 # {FilePath: FileObj}
 #
 gLIBINSTANCEDICT = {}
+
+#
+# Store the list of DIST
+#
+gTO_BE_INSTALLED_DIST_LIST = []

BaseTools/Source/Python/UPT/Logger/StringTable.py

@@ -1,7 +1,7 @@
 ## @file
 # This file is used to define strings used in the UPT tool
 #
-# Copyright (c) 2011 - 2016, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2011 - 2017, Intel Corporation. All rights reserved.<BR>
 #
 # This program and the accompanying materials are licensed and made available 
 # under the terms and conditions of the BSD License which accompanies this 
@@ -785,7 +785,7 @@ WRN_MODULE_EXISTED        = _("This module already exists: %s")
 WRN_FILE_EXISTED          = _("This file already exists: %s")
 WRN_FILE_NOT_OVERWRITTEN  = \
 _("This file already exist and cannot be overwritten: %s")
-WRN_DIST_PKG_INSTALLED = _("This distribution package has previously been installed.")
+WRN_DIST_PKG_INSTALLED = _("This distribution package %s has previously been installed.")
 WRN_DIST_NOT_FOUND         = _(
     "Distribution is not found at location %s")
 WRN_MULTI_PCD_RANGES      = _(

BaseTools/Source/Python/UPT/ReplacePkg.py

@@ -1,7 +1,7 @@
 ## @file
 # Replace distribution package.
 #
-# Copyright (c) 2014, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2014 - 2017, Intel Corporation. All rights reserved.<BR>
 #
 # This program and the accompanying materials are licensed and made available 
 # under the terms and conditions of the BSD License which accompanies this 
@@ -99,9 +99,9 @@ def Main(Options = None):
             DistFile.Close()
         if ContentZipFile:
             ContentZipFile.Close()
-        if GlobalData.gUNPACK_DIR:
+        for TempDir in GlobalData.gUNPACK_DIR:
-            rmtree(GlobalData.gUNPACK_DIR)
+            rmtree(TempDir)
-            GlobalData.gUNPACK_DIR = None
+        GlobalData.gUNPACK_DIR = []
         Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_DONE)        
 
     if ReturnCode == 0:

BaseTools/Source/Python/UPT/TestInstall.py

@@ -1,7 +1,7 @@
 # # @file
 # Test Install distribution package
 #
-# Copyright (c) 2016, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2016 - 2017, Intel Corporation. All rights reserved.<BR>
 #
 # This program and the accompanying materials are licensed and made available
 # under the terms and conditions of the BSD License which accompanies this
@@ -90,9 +90,9 @@ def Main(Options=None):
             DistFile.Close()
         if ContentZipFile:
             ContentZipFile.Close()
-        if GlobalData.gUNPACK_DIR:
+        for TempDir in GlobalData.gUNPACK_DIR:
-            shutil.rmtree(GlobalData.gUNPACK_DIR)
+            shutil.rmtree(TempDir)
-            GlobalData.gUNPACK_DIR = None
+        GlobalData.gUNPACK_DIR = []
         Logger.Quiet(ST.MSG_REMOVE_TEMP_FILE_DONE)
     if ReturnCode == 0:
         Logger.Quiet(ST.MSG_FINISH)

BaseTools/Source/Python/UPT/UPT.py

@@ -120,7 +120,7 @@ def Main():
 
     Parser.add_option("-q", "--quiet", action="store_true", dest="opt_quiet", help=ST.HLP_RETURN_AND_DISPLAY)
 
-    Parser.add_option("-i", "--install", action="store", type="string", dest="Install_Distribution_Package_File",
+    Parser.add_option("-i", "--install", action="append", type="string", dest="Install_Distribution_Package_File",
                       help=ST.HLP_SPECIFY_PACKAGE_NAME_INSTALL)
 
     Parser.add_option("-c", "--create", action="store", type="string", dest="Create_Distribution_Package_File",
@@ -228,12 +228,14 @@ def Main():
             RunModule = MkPkg.Main
 
         elif Opt.PackFileToInstall:
-            if not Opt.PackFileToInstall.endswith('.dist'):
+            AbsPath = []
-                Logger.Error("InstallPkg", FILE_TYPE_MISMATCH, ExtraData=ST.ERR_DIST_EXT_ERROR % Opt.PackFileToInstall)
+            for Item in Opt.PackFileToInstall:
+                if not Item.endswith('.dist'):
+                    Logger.Error("InstallPkg", FILE_TYPE_MISMATCH, ExtraData=ST.ERR_DIST_EXT_ERROR % Item)
 
-            AbsPath = GetFullPathDist(Opt.PackFileToInstall, WorkspaceDir)
+                AbsPath.append(GetFullPathDist(Item, WorkspaceDir))
-            if not AbsPath:
+                if not AbsPath:
-                Logger.Error("InstallPkg", FILE_NOT_FOUND, ST.ERR_INSTALL_DIST_NOT_FOUND % Opt.PackFileToInstall)
+                    Logger.Error("InstallPkg", FILE_NOT_FOUND, ST.ERR_INSTALL_DIST_NOT_FOUND % Item)
 
             Opt.PackFileToInstall = AbsPath
             setattr(Opt, 'PackageFile', Opt.PackFileToInstall)