Shi, Steven 0e7e7a264c BaseTools: Improve the cache hit in the edk2 build cache
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1927

Current cache hash algorithm does not parse and generate
the makefile to get the accurate dependency files for a
module. It instead use the platform and package meta files
to get the module depenedency in a quick but over approximate
way. These meta files are monolithic and involve many redundant
dependency for the module, which cause the module build
cache miss easily.
This patch introduces one more cache checkpoint and a new
hash algorithm besides the current quick one. The new hash
algorithm leverages the module makefile to achieve more
accurate and precise dependency info for a module. When
the build cache miss with the first quick hash, the
Basetool will caculate new one after makefile is generated
and then check again.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Signed-off-by: Steven Shi <steven.shi@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
2019-08-20 14:06:05 +08:00

130 lines
4.0 KiB
Python
Executable File

## @file
# This file is used to define common static strings used by INF/DEC/DSC files
#
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
import re
gIsWindows = None
gWorkspace = "."
gOptions = None
gCaseInsensitive = False
gAllFiles = None
gCommand = None
gSKUID_CMD = None
gGlobalDefines = {}
gPlatformDefines = {}
# PCD name and value pair for fixed at build and feature flag
gPlatformPcds = {}
# PCDs with type that are not fixed at build and feature flag
gPlatformOtherPcds = {}
gActivePlatform = None
gCommandLineDefines = {}
gEdkGlobal = {}
gCommandMaxLength = 4096
# for debug trace purpose when problem occurs
gProcessingFile = ''
gBuildingModule = ''
gSkuids = []
gDefaultStores = []
# definition for a MACRO name. used to create regular expressions below.
_MacroNamePattern = "[A-Z][A-Z0-9_]*"
## Regular expression for matching macro used in DSC/DEC/INF file inclusion
gMacroRefPattern = re.compile("\$\(({})\)".format(_MacroNamePattern), re.UNICODE)
gMacroDefPattern = re.compile("^(DEFINE|EDK_GLOBAL)[ \t]+")
gMacroNamePattern = re.compile("^{}$".format(_MacroNamePattern))
# definition for a GUID. used to create regular expressions below.
_HexChar = r"[0-9a-fA-F]"
_GuidPattern = r"{Hex}{{8}}-{Hex}{{4}}-{Hex}{{4}}-{Hex}{{4}}-{Hex}{{12}}".format(Hex=_HexChar)
## Regular expressions for GUID matching
gGuidPattern = re.compile(r'{}'.format(_GuidPattern))
gGuidPatternEnd = re.compile(r'{}$'.format(_GuidPattern))
## Regular expressions for HEX matching
g4HexChar = re.compile(r'{}{{4}}'.format(_HexChar))
gHexPattern = re.compile(r'0[xX]{}+'.format(_HexChar))
gHexPatternAll = re.compile(r'0[xX]{}+$'.format(_HexChar))
## Regular expressions for string identifier checking
gIdentifierPattern = re.compile('^[a-zA-Z][a-zA-Z0-9_]*$', re.UNICODE)
## Regular expression for GUID c structure format
_GuidCFormatPattern = r"{{\s*0[xX]{Hex}{{1,8}}\s*,\s*0[xX]{Hex}{{1,4}}\s*,\s*0[xX]{Hex}{{1,4}}" \
r"\s*,\s*{{\s*0[xX]{Hex}{{1,2}}\s*,\s*0[xX]{Hex}{{1,2}}" \
r"\s*,\s*0[xX]{Hex}{{1,2}}\s*,\s*0[xX]{Hex}{{1,2}}" \
r"\s*,\s*0[xX]{Hex}{{1,2}}\s*,\s*0[xX]{Hex}{{1,2}}" \
r"\s*,\s*0[xX]{Hex}{{1,2}}\s*,\s*0[xX]{Hex}{{1,2}}\s*}}\s*}}".format(Hex=_HexChar)
gGuidCFormatPattern = re.compile(r"{}".format(_GuidCFormatPattern))
#
# A global variable for whether current build in AutoGen phase or not.
#
gAutoGenPhase = False
#
# The Conf dir outside the workspace dir
#
gConfDirectory = ''
gBuildDirectory = ''
#
# The relative default database file path
#
gDatabasePath = ".cache/build.db"
#
# Build flag for binary build
#
gIgnoreSource = False
#
# FDF parser
#
gFdfParser = None
BuildOptionPcd = []
#
# Mixed PCD name dict
#
MixedPcd = {}
# Structure Pcd dict
gStructurePcd = {}
gPcdSkuOverrides={}
# Pcd name for the Pcd which used in the Conditional directives
gConditionalPcds = []
gUseHashCache = None
gBinCacheDest = None
gBinCacheSource = None
gPlatformHash = None
gPackageHash = {}
gModuleHash = {}
gEnableGenfdsMultiThread = False
gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure
# Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
gModuleBuildTracking = dict()
# Dictionary of booleans that dictate whether a module or
# library can be skiped
# Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: Module\Library Name Value: True\False
gBuildHashSkipTracking = dict()
# Common dictionary to share module cache intermediate result and state
gCacheIR = None
# Common lock for the file access in multiple process AutoGens
file_lock = None
# Common dictionary to share platform libraries' constant Pcd
libConstPcd = None
# Common dictionary to share platform libraries' reference info
Refes = None