BaseTools: AutoGen refactor ModuleAutoGen caching

1) Add a new file Common/caching.py
    a.	Allows for automated caching of repeated class functions, class
        properties, and non-class functions
    b.	When called the first time the value is cached and if called a
        second time, the cached result is called, not the function.
    c.	When used, this saves lots of memory since the actual function
        pointers are replaced with smaller data elements.
    d.  note that not all features are used yet.
2) Fix AutoGen/GenMake and AutoGen/GetC to not call into private member
    variables of ModuleAutoGen class
    a. use the existing accessor properties for the data
3) Change AutoGen classes to remove a exception for duplicate members in
    __new__ and use ?in? testing to speed up
4)	Work on ModuleAutoGen class
    a.	Change all properties that use caching to use @caching_property
        (see #1 above)
    b.	Change all properties that do not use caching to use standard python
        decorator "@property"
    c.	Change all cases where a dictionary/set/list/object was created
        and then immediately updated to use constructor parameters
    d.	Refactor each property function to remove the internal variable
        that is no longer needed (this helps find items like #2 above)
    e.  Refactor _ApplyBuildRule with optional parameter to work around
        circular dependency with BinaryFileList.

Note that 4e was almost certainly unintended as the functions were acting on
incomplete information since they were directly accessing private instance
variables at the same time another function on the stack was creating the same
private isntance data.

This overall changes behavior slightly, but makes the codebase smaller and
easier to read.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Jaben Carsey <jaben.carsey@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>

BaseTools/Source/Python/AutoGen/GenC.py

@@ -1329,7 +1329,7 @@ def CreateLibraryPcdCode(Info, AutoGenC, AutoGenH, Pcd):
         AutoGenH.Append('//#define %s  ASSERT(FALSE)  // It is not allowed to set value for a FIXED_AT_BUILD PCD\n' % SetModeName)
 
         ConstFixedPcd = False
-        if PcdItemType == TAB_PCDS_FIXED_AT_BUILD and (key in Info.ConstPcd or (Info.IsLibrary and not Info._ReferenceModules)):
+        if PcdItemType == TAB_PCDS_FIXED_AT_BUILD and (key in Info.ConstPcd or (Info.IsLibrary and not Info.ReferenceModules)):
             ConstFixedPcd = True
             if key in Info.ConstPcd:
                 Pcd.DefaultValue = Info.ConstPcd[key]

BaseTools/Source/Python/AutoGen/GenMake.py

@@ -798,14 +798,14 @@ cleanlib:
                                     Tool = Flag
                                     break
                         if Tool:
-                            if 'PATH' not in self._AutoGenObject._BuildOption[Tool]:
+                            if 'PATH' not in self._AutoGenObject.BuildOption[Tool]:
                                 EdkLogger.error("build", AUTOGEN_ERROR, "%s_PATH doesn't exist in %s ToolChain and %s Arch." %(Tool, self._AutoGenObject.ToolChain, self._AutoGenObject.Arch), ExtraData="[%s]" % str(self._AutoGenObject))
-                            SingleCommandLength += len(self._AutoGenObject._BuildOption[Tool]['PATH'])
+                            SingleCommandLength += len(self._AutoGenObject.BuildOption[Tool]['PATH'])
                             for item in SingleCommandList[1:]:
                                 if FlagDict[Tool]['Macro'] in item:
-                                    if 'FLAGS' not in self._AutoGenObject._BuildOption[Tool]:
+                                    if 'FLAGS' not in self._AutoGenObject.BuildOption[Tool]:
                                         EdkLogger.error("build", AUTOGEN_ERROR, "%s_FLAGS doesn't exist in %s ToolChain and %s Arch." %(Tool, self._AutoGenObject.ToolChain, self._AutoGenObject.Arch), ExtraData="[%s]" % str(self._AutoGenObject))
-                                    Str = self._AutoGenObject._BuildOption[Tool]['FLAGS']
+                                    Str = self._AutoGenObject.BuildOption[Tool]['FLAGS']
                                     for Option in self._AutoGenObject.BuildOption:
                                         for Attr in self._AutoGenObject.BuildOption[Option]:
                                             if Str.find(Option + '_' + Attr) != -1:
@@ -820,7 +820,7 @@ cleanlib:
                                             break
                                     SingleCommandLength += len(Str)
                                 elif '$(INC)' in item:
-                                    SingleCommandLength += self._AutoGenObject.IncludePathLength + len(IncPrefix) * len(self._AutoGenObject._IncludePathList)
+                                    SingleCommandLength += self._AutoGenObject.IncludePathLength + len(IncPrefix) * len(self._AutoGenObject.IncludePathList)
                                 elif item.find('$(') != -1:
                                     Str = item
                                     for Option in self._AutoGenObject.BuildOption:
@@ -846,7 +846,7 @@ cleanlib:
                         Key = Flag + '_RESP'
                         RespMacro = FlagDict[Flag]['Macro'].replace('FLAGS', 'RESP')
                         Value = self._AutoGenObject.BuildOption[Flag]['FLAGS']
-                        for inc in self._AutoGenObject._IncludePathList:
+                        for inc in self._AutoGenObject.IncludePathList:
                             Value += ' ' + IncPrefix + inc
                         for Option in self._AutoGenObject.BuildOption:
                             for Attr in self._AutoGenObject.BuildOption[Option]:

BaseTools/Source/Python/Common/caching.py

@@ -0,0 +1,47 @@
+## @file
+# help with caching in BaseTools
+#
+# Copyright (c) 2018, Intel Corporation. All rights reserved.<BR>
+#
+# This program and the accompanying materials
+# are licensed and made available under the terms and conditions of the BSD License
+# which accompanies this distribution.  The full text of the license may be found at
+# http://opensource.org/licenses/bsd-license.php
+#
+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+#
+
+## Import Modules
+#
+
+# for class function
+class cached_class_function(object):
+    def __init__(self, function):
+        self._function = function
+    def __get__(self, obj, cls):
+        def CallMeHere(*args,**kwargs):
+            Value = self._function(obj, *args,**kwargs)
+            obj.__dict__[self._function.__name__] = lambda *args,**kwargs:Value
+            return Value
+        return CallMeHere
+
+# for class property
+class cached_property(object):
+    def __init__(self, function):
+        self._function = function
+    def __get__(self, obj, cls):
+        Value = obj.__dict__[self._function.__name__] = self._function(obj)
+        return Value
+
+# for non-class function
+class cached_basic_function(object):
+    def __init__(self, function):
+        self._function = function
+    # wrapper to call _do since <class>.__dict__ doesn't support changing __call__
+    def __call__(self,*args,**kwargs):
+        return self._do(*args,**kwargs)
+    def _do(self,*args,**kwargs):
+        Value = self._function(*args,**kwargs)
+        self.__dict__['_do'] = lambda self,*args,**kwargs:Value
+        return Value