tyler.durden
/
audk
mirror of https://github.com/acidanthera/audk.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BaseTools: Replace the sqlite database with list 

https://bugzilla.tianocore.org/show_bug.cgi?id=1288

[V2]
Optimize this patch so that it can be easy to review.
This patch is just apply the change to original files while
not create new similar files.

[V1]
This patch is one of build tool performance improvement
series patches.

This patch is going to use python list to store the parser data
instead of using sqlite database.

The replacement solution is as below:

SQL insert: list.append()
SQL select: list comprehension. for example:
Select * from table where field = “something”
->
[ item for item in table if item[3] == “something”]

SQL update: python map function. for example:
Update table set field1=newvalue where filed2 = “something”.
-> map(lambda x: x[1] = newvalue,
   [item for item in table if item[2] == “something”])

SQL delete: list comprehension.

With this change, We can save the time of interpreting SQL statement
and the time of write database to file system

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: BobCF <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Jaben Carsey <jaben.carsey@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
Reviewed-by: Jaben Carsey <jaben.carsey@intel.com>
This commit is contained in:
BobCF 2018-11-09 15:41:02 +08:00
parent b3497bad12
commit 2f818ed0fb
6 changed files with 189 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
BaseTools/Source/Python/GenFds/GenFds.py
View File

@ -226,12 +226,11 @@ def GenFdsApi(FdsCommandDict, WorkSpaceDataBase=None):
"""call Workspace build create database""" """call Workspace build create database"""
GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath)) GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
if WorkSpaceDataBase: if WorkSpaceDataBase:
BuildWorkSpace = WorkSpaceDataBase BuildWorkSpace = WorkSpaceDataBase
else: else:
BuildWorkSpace = WorkspaceDatabase(GlobalData.gDatabasePath) BuildWorkSpace = WorkspaceDatabase()
BuildWorkSpace.InitDatabase()
# #
# Get files real name in workspace dir # Get files real name in workspace dir
# #

93
BaseTools/Source/Python/Workspace/MetaDataTable.py
View File

@ -39,12 +39,13 @@ class Table(object):
_ID_MAX_ = 0x80000000 _ID_MAX_ = 0x80000000
_DUMMY_ = 0 _DUMMY_ = 0
def __init__(self, Cursor, Name='', IdBase=0, Temporary=False): def __init__(self, Db, Name='', IdBase=0, Temporary=False):
self.Cur = Cursor self.Db = Db
self.Table = Name self.Table = Name
self.IdBase = int(IdBase) self.IdBase = int(IdBase)
self.ID = int(IdBase) self.ID = int(IdBase)
self.Temporary = Temporary self.Temporary = Temporary
self.Contents = []
def __str__(self): def __str__(self):
return self.Table return self.Table
@ -54,15 +55,7 @@ class Table(object):
# Create a table # Create a table
# #
def Create(self, NewTable=True): def Create(self, NewTable=True):
if NewTable: self.Db.CreateEmptyTable(self.Table)
self.Drop()
if self.Temporary:
SqlCommand = """create temp table IF NOT EXISTS %s (%s)""" % (self.Table, self._COLUMN_)
else:
SqlCommand = """create table IF NOT EXISTS %s (%s)""" % (self.Table, self._COLUMN_)
EdkLogger.debug(EdkLogger.DEBUG_8, SqlCommand)
self.Cur.execute(SqlCommand)
self.ID = self.GetId() self.ID = self.GetId()
## Insert table ## Insert table
@ -73,30 +66,12 @@ class Table(object):
self.ID = self.ID + self._ID_STEP_ self.ID = self.ID + self._ID_STEP_
if self.ID >= (self.IdBase + self._ID_MAX_): if self.ID >= (self.IdBase + self._ID_MAX_):
self.ID = self.IdBase + self._ID_STEP_ self.ID = self.IdBase + self._ID_STEP_
Values = ", ".join(str(Arg) for Arg in Args) row = [self.ID]
SqlCommand = "insert into %s values(%s, %s)" % (self.Table, self.ID, Values) row.extend(Args)
EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand) self.Contents.append(row)
self.Cur.execute(SqlCommand)
return self.ID return self.ID
## Query table
#
# Query all records of the table
#
def Query(self):
SqlCommand = """select * from %s""" % self.Table
self.Cur.execute(SqlCommand)
for Rs in self.Cur:
EdkLogger.verbose(str(Rs))
TotalCount = self.GetId()
## Drop a table
#
# Drop the table
#
def Drop(self):
SqlCommand = """drop table IF EXISTS %s""" % self.Table
self.Cur.execute(SqlCommand)
## Get count ## Get count
# #
@ -105,14 +80,13 @@ class Table(object):
# @retval Count: Total count of all records # @retval Count: Total count of all records
# #
def GetCount(self): def GetCount(self):
SqlCommand = """select count(ID) from %s""" % self.Table tab = self.Db.GetTable(self.Table)
Record = self.Cur.execute(SqlCommand).fetchall() return len(tab)
return Record[0][0]
def GetId(self): def GetId(self):
SqlCommand = """select max(ID) from %s""" % self.Table tab = self.Db.GetTable(self.Table)
Record = self.Cur.execute(SqlCommand).fetchall() Id = max([int(item[0]) for item in tab])
Id = Record[0][0]
if Id is None: if Id is None:
Id = self.IdBase Id = self.IdBase
return Id return Id
@ -134,25 +108,26 @@ class Table(object):
# #
def Exec(self, SqlCommand): def Exec(self, SqlCommand):
EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand) EdkLogger.debug(EdkLogger.DEBUG_5, SqlCommand)
self.Cur.execute(SqlCommand) self.Db.execute(SqlCommand)
RecordSet = self.Cur.fetchall() RecordSet = self.Db.fetchall()
return RecordSet return RecordSet
def SetEndFlag(self): def SetEndFlag(self):
self.Exec("insert into %s values(%s)" % (self.Table, self._DUMMY_)) Tab = self.Db.GetTable(self.Table)
# Tab.append(self._DUMMY_)
# Need to execution commit for table data changed.
#
self.Cur.connection.commit()
def IsIntegral(self): def IsIntegral(self):
Result = self.Exec("select min(ID) from %s" % (self.Table)) tab = self.Db.GetTable(self.Table)
if Result[0][0] != -1: Id = min([int(item[0]) for item in tab])
if Id != -1:
return False return False
return True return True
def GetAll(self): def GetAll(self):
return self.Exec("select * from %s where ID > 0 order by ID" % (self.Table)) tab = self.Db.GetTable(self.Table)
return tab
## TableFile ## TableFile
# #
@ -227,22 +202,6 @@ class TableFile(Table):
File.TimeStamp File.TimeStamp
) )
## Get ID of a given file
#
# @param FilePath Path of file
#
# @retval ID ID value of given file in the table
#
def GetFileId(self, File, FromItem=None):
if FromItem:
QueryScript = "select ID from %s where FullPath = '%s' and FromItem = %s" % (self.Table, str(File), str(FromItem))
else:
QueryScript = "select ID from %s where FullPath = '%s'" % (self.Table, str(File))
RecordList = self.Exec(QueryScript)
if len(RecordList) == 0:
return None
return RecordList[0][0]
## Get type of a given file ## Get type of a given file
# #
# @param FileId ID of a file # @param FileId ID of a file
@ -345,8 +304,8 @@ class TableDataModel(Table):
def GetCrossIndex(self, ModelName): def GetCrossIndex(self, ModelName):
CrossIndex = -1 CrossIndex = -1
SqlCommand = """select CrossIndex from DataModel where name = '""" + ModelName + """'""" SqlCommand = """select CrossIndex from DataModel where name = '""" + ModelName + """'"""
self.Cur.execute(SqlCommand) self.Db.execute(SqlCommand)
for Item in self.Cur: for Item in self.Db:
CrossIndex = Item[0] CrossIndex = Item[0]
return CrossIndex return CrossIndex

5
BaseTools/Source/Python/Workspace/MetaFileParser.py
View File

@ -1372,8 +1372,7 @@ class DscParser(MetaFileParser):
MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR : self._ProcessError, MODEL_META_DATA_CONDITIONAL_STATEMENT_ERROR : self._ProcessError,
} }
self._Table = MetaFileStorage(self._RawTable.Cur, self.MetaFile, MODEL_FILE_DSC, True) self._Table = MetaFileStorage(self._RawTable.DB, self.MetaFile, MODEL_FILE_DSC, True)
self._Table.Create()
self._DirectiveStack = [] self._DirectiveStack = []
self._DirectiveEvalStack = [] self._DirectiveEvalStack = []
self._FileWithError = self.MetaFile self._FileWithError = self.MetaFile
@ -1632,7 +1631,7 @@ class DscParser(MetaFileParser):
Owner = self._Content[self._ContentIndex - 1][8] Owner = self._Content[self._ContentIndex - 1][8]
else: else:
Owner = self._Content[self._ContentIndex - 1][0] Owner = self._Content[self._ContentIndex - 1][0]
IncludedFileTable = MetaFileStorage(self._Table.Cur, IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem) IncludedFileTable = MetaFileStorage(self._RawTable.DB, IncludedFile1, MODEL_FILE_DSC, False, FromItem=FromItem)
Parser = DscParser(IncludedFile1, self._FileType, self._Arch, IncludedFileTable, Parser = DscParser(IncludedFile1, self._FileType, self._Arch, IncludedFileTable,
Owner=Owner, From=FromItem) Owner=Owner, From=FromItem)

252
BaseTools/Source/Python/Workspace/MetaFileTable.py
View File

@ -20,55 +20,60 @@ import uuid
import Common.EdkLogger as EdkLogger import Common.EdkLogger as EdkLogger
from Common.BuildToolError import FORMAT_INVALID from Common.BuildToolError import FORMAT_INVALID
from .MetaDataTable import Table, TableFile
from .MetaDataTable import ConvertToSqlString
from CommonDataClass.DataClass import MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE_INF, \ from CommonDataClass.DataClass import MODEL_FILE_DSC, MODEL_FILE_DEC, MODEL_FILE_INF, \
MODEL_FILE_OTHERS MODEL_FILE_OTHERS
from Common.DataType import * from Common.DataType import *
class MetaFileTable(Table): class MetaFileTable():
# TRICK: use file ID as the part before '.' # TRICK: use file ID as the part before '.'
_ID_STEP_ = 0.00000001 _ID_STEP_ = 0.00000001
_ID_MAX_ = 0.99999999 _ID_MAX_ = 0.99999999
## Constructor ## Constructor
def __init__(self, Cursor, MetaFile, FileType, Temporary, FromItem=None): def __init__(self, DB, MetaFile, FileType, Temporary, FromItem=None):
self.MetaFile = MetaFile self.MetaFile = MetaFile
self.TableName = ""
self._FileIndexTable = TableFile(Cursor) self.DB = DB
self._FileIndexTable.Create(False) self._NumpyTab = None
self.FileId = len(DB.TblFile)
FileId = self._FileIndexTable.GetFileId(MetaFile, FromItem) self.ID = self.FileId
if not FileId: self.CurrentContent = []
FileId = self._FileIndexTable.InsertFile(MetaFile, FileType, FromItem) DB.TblFile.append([MetaFile.Name,
MetaFile.Ext,
MetaFile.Dir,
MetaFile.Path,
FileType,
MetaFile.TimeStamp,
FromItem])
if Temporary: if Temporary:
TableName = "_%s_%s_%s" % (FileType, FileId, uuid.uuid4().hex) self.TableName = "_%s_%s_%s" % (FileType, len(DB.TblFile), uuid.uuid4().hex)
else: else:
TableName = "_%s_%s" % (FileType, FileId) self.TableName = "_%s_%s" % (FileType, len(DB.TblFile))
#Table.__init__(self, Cursor, TableName, FileId, False)
Table.__init__(self, Cursor, TableName, FileId, Temporary)
self.Create(not self.IsIntegrity())
def IsIntegrity(self): def IsIntegrity(self):
try: try:
TimeStamp = self.MetaFile.TimeStamp TimeStamp = self.MetaFile.TimeStamp
Result = self.Cur.execute("select ID from %s where ID<0" % (self.Table)).fetchall() Result = int(self.CurrentContent[-1][0]) < 0
if not Result: if not Result:
# update the timestamp in database # update the timestamp in database
self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp) self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
return False return False
if TimeStamp != self._FileIndexTable.GetFileTimeStamp(self.IdBase): if TimeStamp != self.DB.GetFileTimeStamp(self.FileId):
# update the timestamp in database # update the timestamp in database
self._FileIndexTable.SetFileTimeStamp(self.IdBase, TimeStamp) self.DB.SetFileTimeStamp(self.FileId, TimeStamp)
return False return False
except Exception as Exc: except Exception as Exc:
EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc)) EdkLogger.debug(EdkLogger.DEBUG_5, str(Exc))
return False return False
return True return True
def SetEndFlag(self):
self.CurrentContent.append(self._DUMMY_)
def GetAll(self):
return [item for item in self.CurrentContent if item[0] > 0 ]
## Python class representation of table storing module data ## Python class representation of table storing module data
class ModuleTable(MetaFileTable): class ModuleTable(MetaFileTable):
_ID_STEP_ = 0.00000001 _ID_STEP_ = 0.00000001
@ -89,11 +94,11 @@ class ModuleTable(MetaFileTable):
Enabled INTEGER DEFAULT 0 Enabled INTEGER DEFAULT 0
''' '''
# used as table end flag, in case the changes to database is not committed to db file # used as table end flag, in case the changes to database is not committed to db file
_DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1" _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1]
## Constructor ## Constructor
def __init__(self, Cursor, MetaFile, Temporary): def __init__(self, Db, MetaFile, Temporary):
MetaFileTable.__init__(self, Cursor, MetaFile, MODEL_FILE_INF, Temporary) MetaFileTable.__init__(self, Db, MetaFile, MODEL_FILE_INF, Temporary)
## Insert a record into table Inf ## Insert a record into table Inf
# #
@ -112,22 +117,29 @@ class ModuleTable(MetaFileTable):
# #
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0): BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0):
(Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2))
return Table.Insert( (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
self, self.ID = self.ID + self._ID_STEP_
Model, if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
Value1, self.ID = MODEL_FILE_INF + self._ID_STEP_
Value2,
Value3, row = [ self.ID,
Scope1, Model,
Scope2, Value1,
BelongsToItem, Value2,
StartLine, Value3,
StartColumn, Scope1,
EndLine, Scope2,
EndColumn, BelongsToItem,
Enabled StartLine,
) StartColumn,
EndLine,
EndColumn,
Enabled
]
self.CurrentContent.append(row)
return self.ID
## Query table ## Query table
# #
@ -138,18 +150,25 @@ class ModuleTable(MetaFileTable):
# @retval: A recordSet of all found records # @retval: A recordSet of all found records
# #
def Query(self, Model, Arch=None, Platform=None, BelongsToItem=None): def Query(self, Model, Arch=None, Platform=None, BelongsToItem=None):
ConditionString = "Model=%s AND Enabled>=0" % Model
ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine" QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>=0 ]
if Arch is not None and Arch != TAB_ARCH_COMMON: if Arch is not None and Arch != TAB_ARCH_COMMON:
ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Arch ArchList = set(['COMMON'])
if Platform is not None and Platform != TAB_COMMON: ArchList.add(Arch)
ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT')" % Platform result = [item for item in result if item[5] in ArchList]
if BelongsToItem is not None:
ConditionString += " AND BelongsToItem=%s" % BelongsToItem
SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString) if Platform is not None and Platform != TAB_COMMON:
return self.Exec(SqlCommand) Platformlist = set( ['COMMON','DEFAULT'])
Platformlist.add(Platform)
result = [item for item in result if item[6] in Platformlist]
if BelongsToItem is not None:
result = [item for item in result if item[7] == BelongsToItem]
result = [ [r[2],r[3],r[4],r[5],r[6],r[0],r[9]] for r in result ]
return result
## Python class representation of table storing package data ## Python class representation of table storing package data
class PackageTable(MetaFileTable): class PackageTable(MetaFileTable):
@ -169,7 +188,7 @@ class PackageTable(MetaFileTable):
Enabled INTEGER DEFAULT 0 Enabled INTEGER DEFAULT 0
''' '''
# used as table end flag, in case the changes to database is not committed to db file # used as table end flag, in case the changes to database is not committed to db file
_DUMMY_ = "-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1" _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====', -1, -1, -1, -1, -1, -1]
## Constructor ## Constructor
def __init__(self, Cursor, MetaFile, Temporary): def __init__(self, Cursor, MetaFile, Temporary):
@ -194,22 +213,27 @@ class PackageTable(MetaFileTable):
# #
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON,
BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0): BelongsToItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=0):
(Value1, Value2, Value3, Scope1, Scope2) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2)) (Value1, Value2, Value3, Scope1, Scope2) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip())
return Table.Insert( self.ID = self.ID + self._ID_STEP_
self, if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
Model, self.ID = MODEL_FILE_INF + self._ID_STEP_
Value1,
Value2, row = [ self.ID,
Value3, Model,
Scope1, Value1,
Scope2, Value2,
BelongsToItem, Value3,
StartLine, Scope1,
StartColumn, Scope2,
EndLine, BelongsToItem,
EndColumn, StartLine,
Enabled StartColumn,
) EndLine,
EndColumn,
Enabled
]
self.CurrentContent.append(row)
return self.ID
## Query table ## Query table
# #
@ -219,23 +243,26 @@ class PackageTable(MetaFileTable):
# @retval: A recordSet of all found records # @retval: A recordSet of all found records
# #
def Query(self, Model, Arch=None): def Query(self, Model, Arch=None):
ConditionString = "Model=%s AND Enabled>=0" % Model
ValueString = "Value1,Value2,Value3,Scope1,Scope2,ID,StartLine" QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>=0 ]
if Arch is not None and Arch != TAB_ARCH_COMMON: if Arch is not None and Arch != TAB_ARCH_COMMON:
ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Arch ArchList = set(['COMMON'])
ArchList.add(Arch)
result = [item for item in result if item[5] in ArchList]
SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString) return [[r[2], r[3], r[4], r[5], r[6], r[0], r[8]] for r in result]
return self.Exec(SqlCommand)
def GetValidExpression(self, TokenSpaceGuid, PcdCName): def GetValidExpression(self, TokenSpaceGuid, PcdCName):
SqlCommand = "select Value1,StartLine from %s WHERE Value2='%s' and Value3='%s'" % (self.Table, TokenSpaceGuid, PcdCName)
self.Cur.execute(SqlCommand) QueryTab = self.CurrentContent
result = [[item[2], item[8]] for item in QueryTab if item[3] == TokenSpaceGuid and item[4] == PcdCName]
validateranges = [] validateranges = []
validlists = [] validlists = []
expressions = [] expressions = []
try: try:
for row in self.Cur: for row in result:
comment = row[0] comment = row[0]
LineNum = row[1] LineNum = row[1]
@ -283,7 +310,7 @@ class PlatformTable(MetaFileTable):
Enabled INTEGER DEFAULT 0 Enabled INTEGER DEFAULT 0
''' '''
# used as table end flag, in case the changes to database is not committed to db file # used as table end flag, in case the changes to database is not committed to db file
_DUMMY_ = "-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1, -1, -1, -1, -1" _DUMMY_ = [-1, -1, '====', '====', '====', '====', '====','====', -1, -1, -1, -1, -1, -1, -1]
## Constructor ## Constructor
def __init__(self, Cursor, MetaFile, Temporary, FromItem=0): def __init__(self, Cursor, MetaFile, Temporary, FromItem=0):
@ -309,24 +336,30 @@ class PlatformTable(MetaFileTable):
# #
def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1, def Insert(self, Model, Value1, Value2, Value3, Scope1=TAB_ARCH_COMMON, Scope2=TAB_COMMON, Scope3=TAB_DEFAULT_STORES_DEFAULT,BelongsToItem=-1,
FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=1): FromItem=-1, StartLine=-1, StartColumn=-1, EndLine=-1, EndColumn=-1, Enabled=1):
(Value1, Value2, Value3, Scope1, Scope2, Scope3) = ConvertToSqlString((Value1, Value2, Value3, Scope1, Scope2, Scope3)) (Value1, Value2, Value3, Scope1, Scope2, Scope3) = (Value1.strip(), Value2.strip(), Value3.strip(), Scope1.strip(), Scope2.strip(), Scope3.strip())
return Table.Insert( self.ID = self.ID + self._ID_STEP_
self, if self.ID >= (MODEL_FILE_INF + self._ID_MAX_):
Model, self.ID = MODEL_FILE_INF + self._ID_STEP_
Value1,
Value2, row = [ self.ID,
Value3, Model,
Scope1, Value1,
Scope2, Value2,
Scope3, Value3,
BelongsToItem, Scope1,
FromItem, Scope2,
StartLine, Scope3,
StartColumn, BelongsToItem,
EndLine, FromItem,
EndColumn, StartLine,
Enabled StartColumn,
) EndLine,
EndColumn,
Enabled
]
self.CurrentContent.append(row)
return self.ID
## Query table ## Query table
# #
@ -339,30 +372,33 @@ class PlatformTable(MetaFileTable):
# @retval: A recordSet of all found records # @retval: A recordSet of all found records
# #
def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None): def Query(self, Model, Scope1=None, Scope2=None, BelongsToItem=None, FromItem=None):
ConditionString = "Model=%s AND Enabled>0" % Model
ValueString = "Value1,Value2,Value3,Scope1,Scope2,Scope3,ID,StartLine" QueryTab = self.CurrentContent
result = [item for item in QueryTab if item[1] == Model and item[-1]>0 ]
if Scope1 is not None and Scope1 != TAB_ARCH_COMMON: if Scope1 is not None and Scope1 != TAB_ARCH_COMMON:
ConditionString += " AND (Scope1='%s' OR Scope1='COMMON')" % Scope1 Sc1 = set(['COMMON'])
if Scope2 is not None and Scope2 != TAB_COMMON: Sc1.add(Scope1)
# Cover the case that CodeBase is 'COMMON' for BuildOptions section result = [item for item in result if item[5] in Sc1]
Sc2 = set( ['COMMON','DEFAULT'])
if Scope2 and Scope2 != TAB_COMMON:
if '.' in Scope2: if '.' in Scope2:
Index = Scope2.index('.') Index = Scope2.index('.')
NewScope = TAB_COMMON + Scope2[Index:] NewScope = TAB_COMMON + Scope2[Index:]
ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT' OR Scope2='%s')" % (Scope2, NewScope) Sc2.add(NewScope)
else: Sc2.add(Scope2)
ConditionString += " AND (Scope2='%s' OR Scope2='COMMON' OR Scope2='DEFAULT')" % Scope2 result = [item for item in result if item[6] in Sc2]
if BelongsToItem is not None: if BelongsToItem is not None:
ConditionString += " AND BelongsToItem=%s" % BelongsToItem result = [item for item in result if item[8] == BelongsToItem]
else: else:
ConditionString += " AND BelongsToItem<0" result = [item for item in result if item[8] < 0]
if FromItem is not None: if FromItem is not None:
ConditionString += " AND FromItem=%s" % FromItem result = [item for item in result if item[9] == FromItem]
result = [ [r[2],r[3],r[4],r[5],r[6],r[7],r[0],r[9]] for r in result ]
return result
SqlCommand = "SELECT %s FROM %s WHERE %s" % (ValueString, self.Table, ConditionString)
return self.Exec(SqlCommand)
## Factory class to produce different storage for different type of meta-file ## Factory class to produce different storage for different type of meta-file
class MetaFileStorage(object): class MetaFileStorage(object):

131
BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
View File

@ -16,7 +16,6 @@
# Import Modules # Import Modules
# #
from __future__ import absolute_import from __future__ import absolute_import
import sqlite3
from Common.StringUtils import * from Common.StringUtils import *
from Common.DataType import * from Common.DataType import *
from Common.Misc import * from Common.Misc import *
@ -122,7 +121,7 @@ class WorkspaceDatabase(object):
FilePath, FilePath,
FileType, FileType,
Arch, Arch,
MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType) MetaFileStorage(self.WorkspaceDb, FilePath, FileType)
) )
# alwasy do post-process, in case of macros change # alwasy do post-process, in case of macros change
MetaFile.DoPostProcess() MetaFile.DoPostProcess()
@ -152,133 +151,23 @@ class WorkspaceDatabase(object):
# @param GlobalMacros Global macros used for replacement during file parsing # @param GlobalMacros Global macros used for replacement during file parsing
# @prarm RenewDb=False Create new database file if it's already there # @prarm RenewDb=False Create new database file if it's already there
# #
def __init__(self, DbPath, RenewDb=False): def __init__(self):
self._DbClosedFlag = False self.DB = dict()
if not DbPath:
DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
# don't create necessary path for db in memory
if DbPath != ':memory:':
DbDir = os.path.split(DbPath)[0]
if not os.path.exists(DbDir):
os.makedirs(DbDir)
# remove db file in case inconsistency between db and file in file system
if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
os.remove(DbPath)
# create db with optimized parameters
self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
self.Conn.execute("PRAGMA synchronous=OFF")
self.Conn.execute("PRAGMA temp_store=MEMORY")
self.Conn.execute("PRAGMA count_changes=OFF")
self.Conn.execute("PRAGMA cache_size=8192")
#self.Conn.execute("PRAGMA page_size=8192")
# to avoid non-ascii character conversion issue
self.Conn.text_factory = str
self.Cur = self.Conn.cursor()
# create table for internal uses # create table for internal uses
self.TblDataModel = TableDataModel(self.Cur) self.TblDataModel = DataClass.MODEL_LIST
self.TblFile = TableFile(self.Cur) self.TblFile = []
self.Platform = None self.Platform = None
# conversion object for build or file format conversion purpose # conversion object for build or file format conversion purpose
self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self) self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self) self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
## Check whether workspace database need to be renew. def SetFileTimeStamp(self,FileId,TimeStamp):
# The renew reason maybe: self.TblFile[FileId][6] = TimeStamp
# 1) If user force to renew;
# 2) If user do not force renew, and
# a) If the time of last modified python source is newer than database file;
# b) If the time of last modified frozen executable file is newer than database file;
#
# @param force User force renew database
# @param DbPath The absolute path of workspace database file
#
# @return Bool value for whether need renew workspace databse
#
def _CheckWhetherDbNeedRenew (self, force, DbPath):
# if database does not exist, we need do nothing
if not os.path.exists(DbPath): return False
# if user force to renew database, then not check whether database is out of date def GetFileTimeStamp(self,FileId):
if force: return True return self.TblFile[FileId][6]
#
# Check the time of last modified source file or build.exe
# if is newer than time of database, then database need to be re-created.
#
timeOfToolModified = 0
if hasattr(sys, "frozen"):
exePath = os.path.abspath(sys.executable)
timeOfToolModified = os.stat(exePath).st_mtime
else:
curPath = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
if rootPath == "" or rootPath is None:
EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
determine whether database file is out of date!\n")
# walk the root path of source or build's binary to get the time last modified.
for root, dirs, files in os.walk (rootPath):
for dir in dirs:
# bypass source control folder
if dir.lower() in [".svn", "_svn", "cvs"]:
dirs.remove(dir)
for file in files:
ext = os.path.splitext(file)[1]
if ext.lower() == ".py": # only check .py files
fd = os.stat(os.path.join(root, file))
if timeOfToolModified < fd.st_mtime:
timeOfToolModified = fd.st_mtime
if timeOfToolModified > os.stat(DbPath).st_mtime:
EdkLogger.verbose("\nWorkspace database is out of data!")
return True
return False
## Initialize build database
def InitDatabase(self):
EdkLogger.verbose("\nInitialize build database started ...")
#
# Create new tables
#
self.TblDataModel.Create(False)
self.TblFile.Create(False)
#
# Initialize table DataModel
#
self.TblDataModel.InitTable()
EdkLogger.verbose("Initialize build database ... DONE!")
## Query a table
#
# @param Table: The instance of the table to be queried
#
def QueryTable(self, Table):
Table.Query()
def __del__(self):
self.Close()
## Close entire database
#
# Commit all first
# Close the connection and cursor
#
def Close(self):
if not self._DbClosedFlag:
self.Conn.commit()
self.Cur.close()
self.Conn.close()
self._DbClosedFlag = True
## Summarize all packages in the database ## Summarize all packages in the database
def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag): def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
@ -307,7 +196,7 @@ determine whether database file is out of date!\n")
## Summarize all platforms in the database ## Summarize all platforms in the database
def PlatformList(self): def PlatformList(self):
RetVal = [] RetVal = []
for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC): for PlatformFile in [item[3] for item in self.TblFile if item[5] == MODEL_FILE_DSC]:
try: try:
RetVal.append(self.BuildObject[PathClass(PlatformFile), TAB_COMMON]) RetVal.append(self.BuildObject[PathClass(PlatformFile), TAB_COMMON])
except: except:

27
BaseTools/Source/Python/build/build.py
View File

@ -42,7 +42,7 @@ from Common.DataType import *
from Common.BuildVersion import gBUILD_VERSION from Common.BuildVersion import gBUILD_VERSION
from AutoGen.AutoGen import * from AutoGen.AutoGen import *
from Common.BuildToolError import * from Common.BuildToolError import *
from Workspace.WorkspaceDatabase import * from Workspace.WorkspaceDatabase import WorkspaceDatabase
from Common.MultipleWorkspace import MultipleWorkspace as mws from Common.MultipleWorkspace import MultipleWorkspace as mws
from BuildReport import BuildReport from BuildReport import BuildReport
@ -826,10 +826,7 @@ class Build():
GlobalData.gConfDirectory = ConfDirectoryPath GlobalData.gConfDirectory = ConfDirectoryPath
GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath)) GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
if BuildOptions.DisableCache: self.Db = WorkspaceDatabase()
self.Db = WorkspaceDatabase(":memory:")
else:
self.Db = WorkspaceDatabase(GlobalData.gDatabasePath, self.Reparse)
self.BuildDatabase = self.Db.BuildObject self.BuildDatabase = self.Db.BuildObject
self.Platform = None self.Platform = None
self.ToolChainFamily = None self.ToolChainFamily = None
@ -982,9 +979,6 @@ class Build():
if ErrorCode != 0: if ErrorCode != 0:
EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
# create metafile database
if not self.Db_Flag:
self.Db.InitDatabase()
def InitPreBuild(self): def InitPreBuild(self):
self.LoadConfiguration() self.LoadConfiguration()
@ -1003,7 +997,6 @@ class Build():
if 'PREBUILD' in GlobalData.gCommandLineDefines: if 'PREBUILD' in GlobalData.gCommandLineDefines:
self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD') self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD')
else: else:
self.Db.InitDatabase()
self.Db_Flag = True self.Db_Flag = True
Platform = self.Db.MapPlatform(str(self.PlatformFile)) Platform = self.Db.MapPlatform(str(self.PlatformFile))
self.Prebuild = str(Platform.Prebuild) self.Prebuild = str(Platform.Prebuild)
@ -2082,13 +2075,7 @@ class Build():
self.MakeTime += int(round((time.time() - MakeStart))) self.MakeTime += int(round((time.time() - MakeStart)))
MakeContiue = time.time() MakeContiue = time.time()
#
# Save temp tables to a TmpTableDict.
#
for Key in Wa.BuildDatabase._CACHE_:
if Wa.BuildDatabase._CACHE_[Key]._RawData and Wa.BuildDatabase._CACHE_[Key]._RawData._Table and Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
if TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table):
TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] = Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
# #
# #
# All modules have been put in build tasks queue. Tell task scheduler # All modules have been put in build tasks queue. Tell task scheduler
@ -2230,7 +2217,6 @@ class Build():
self._BuildModule() self._BuildModule()
if self.Target == 'cleanall': if self.Target == 'cleanall':
self.Db.Close()
RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True) RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
def CreateAsBuiltInf(self): def CreateAsBuiltInf(self):
@ -2491,10 +2477,7 @@ def Main():
GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList) GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)): if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
MyBuild.Launch() MyBuild.Launch()
# Drop temp tables to avoid database locked.
for TmpTableName in TmpTableDict:
SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
TmpTableDict[TmpTableName].execute(SqlCommand)
#MyBuild.DumpBuildData() #MyBuild.DumpBuildData()
# #
# All job done, no error found and no exception raised # All job done, no error found and no exception raised
@ -2566,7 +2549,7 @@ def Main():
if MyBuild is not None: if MyBuild is not None:
if not BuildError: if not BuildError:
MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime)) MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
MyBuild.Db.Close()
EdkLogger.SetLevel(EdkLogger.QUIET) EdkLogger.SetLevel(EdkLogger.QUIET)
EdkLogger.quiet("\n- %s -" % Conclusion) EdkLogger.quiet("\n- %s -" % Conclusion)
EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime())) EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))