tyler.durden/audk
1
0
Fork 0
mirror of https://github.com/acidanthera/audk.git synced 2025-10-31 03:03:46 +01:00
Code Issues Packages Projects Releases Wiki Activity
audk/BaseTools/Source/Python/AutoGen/StrGather.py
Max Knutsen 307e1650be BaseTools:Add extra debugging message 
BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2014

Add extra debugging to improve error identification.
Error while processing file if the file is read incorrectly

This patch is going to fix that issue.

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
Reviewed-by: Philippe Mathieu-Daude <philmd@redhat.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
2019-08-01 15:36:48 +08:00

631 lines
22 KiB
Python
Raw Blame History

## @file
# This file is used to parse a strings file and create or add to a string database
# file.
#
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
# Import Modules
#
from __future__ import absolute_import
import re
import Common.EdkLogger as EdkLogger
from Common.BuildToolError import *
from .UniClassObject import *
from io import BytesIO
from struct import pack, unpack
from Common.LongFilePathSupport import OpenLongFilePath as open
##
# Static definitions
#
EFI_HII_SIBT_END = '0x00'
EFI_HII_SIBT_STRING_SCSU = '0x10'
EFI_HII_SIBT_STRING_SCSU_FONT = '0x11'
EFI_HII_SIBT_STRINGS_SCSU = '0x12'
EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13'
EFI_HII_SIBT_STRING_UCS2 = '0x14'
EFI_HII_SIBT_STRING_UCS2_FONT = '0x15'
EFI_HII_SIBT_STRINGS_UCS2 = '0x16'
EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17'
EFI_HII_SIBT_DUPLICATE = '0x20'
EFI_HII_SIBT_SKIP2 = '0x21'
EFI_HII_SIBT_SKIP1 = '0x22'
EFI_HII_SIBT_EXT1 = '0x30'
EFI_HII_SIBT_EXT2 = '0x31'
EFI_HII_SIBT_EXT4 = '0x32'
EFI_HII_SIBT_FONT = '0x40'
EFI_HII_PACKAGE_STRINGS = '0x04'
EFI_HII_PACKAGE_FORM = '0x02'
StringPackageType = EFI_HII_PACKAGE_STRINGS
StringPackageForm = EFI_HII_PACKAGE_FORM
StringBlockType = EFI_HII_SIBT_STRING_UCS2
StringSkipType = EFI_HII_SIBT_SKIP2
HexHeader = '0x'
COMMENT = '// '
DEFINE_STR = '#define'
COMMENT_DEFINE_STR = COMMENT + DEFINE_STR
NOT_REFERENCED = 'not referenced'
COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED
CHAR_ARRAY_DEFIN = 'unsigned char'
COMMON_FILE_NAME = 'Strings'
STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
EFI_HII_ARRAY_SIZE_LENGTH = 4
EFI_HII_PACKAGE_HEADER_LENGTH = 4
EFI_HII_HDR_SIZE_LENGTH = 4
EFI_HII_STRING_OFFSET_LENGTH = 4
EFI_STRING_ID = 1
EFI_STRING_ID_LENGTH = 2
EFI_HII_LANGUAGE_WINDOW = 0
EFI_HII_LANGUAGE_WINDOW_LENGTH = 2
EFI_HII_LANGUAGE_WINDOW_NUMBER = 16
EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH
H_C_FILE_HEADER = ['//', \
'// DO NOT EDIT -- auto-generated file', \
'//', \
'// This file is generated by the StrGather utility', \
'//']
LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME'
PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME'
## Convert a dec number to a hex string
#
# Convert a dec number to a formatted hex string in length digit
# The digit is set to default 8
# The hex string starts with "0x"
# DecToHexStr(1000) is '0x000003E8'
# DecToHexStr(1000, 6) is '0x0003E8'
#
# @param Dec: The number in dec format
# @param Digit: The needed digit of hex string
#
# @retval: The formatted hex string
#
def DecToHexStr(Dec, Digit = 8):
return '0x{0:0{1}X}'.format(Dec, Digit)
## Convert a dec number to a hex list
#
# Convert a dec number to a formatted hex list in size digit
# The digit is set to default 8
# DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00']
# DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00']
#
# @param Dec: The number in dec format
# @param Digit: The needed digit of hex list
#
# @retval: A list for formatted hex string
#
def DecToHexList(Dec, Digit = 8):
Hex = '{0:0{1}X}'.format(Dec, Digit)
return ["0x" + Hex[Bit:Bit + 2] for Bit in range(Digit - 2, -1, -2)]
## Convert a acsii string to a hex list
#
# Convert a acsii string to a formatted hex list
# AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53']
#
# @param Ascii: The acsii string
#
# @retval: A list for formatted hex string
#
def AscToHexList(Ascii):
try:
return ['0x{0:02X}'.format(Item) for Item in Ascii]
except:
return ['0x{0:02X}'.format(ord(Item)) for Item in Ascii]
## Create content of .h file
#
# Create content of .h file
#
# @param BaseName: The basename of strings
# @param UniObjectClass A UniObjectClass instance
# @param IsCompatibleMode Compatible mode
# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True
#
# @retval Str: A string of .h file content
#
def CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
Str = []
ValueStartPtr = 60
Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED
Str = WriteLine(Str, Line)
Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED
Str = WriteLine(Str, Line)
UnusedStr = ''
#Group the referred/Unused STRING token together.
for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
Name = StringItem.StringName
Token = StringItem.Token
Referenced = StringItem.Referenced
if Name is not None:
Line = ''
if Referenced == True:
if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4)
else:
Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)
Str = WriteLine(Str, Line)
else:
if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
else:
Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
UnusedStr = WriteLine(UnusedStr, Line)
Str.extend( UnusedStr)
Str = WriteLine(Str, '')
if IsCompatibleMode or UniGenCFlag:
Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')
return "".join(Str)
## Create a complete .h file
#
# Create a complet .h file with file header and file content
#
# @param BaseName: The basename of strings
# @param UniObjectClass A UniObjectClass instance
# @param IsCompatibleMode Compatible mode
# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True
#
# @retval Str: A string of complete .h file
#
def CreateHFile(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag))
return "".join(HFile)
## Create a buffer to store all items in an array
#
# @param BinBuffer Buffer to contain Binary data.
# @param Array: The array need to be formatted
#
def CreateBinBuffer(BinBuffer, Array):
for Item in Array:
BinBuffer.write(pack("B", int(Item, 16)))
## Create a formatted string all items in an array
#
# Use ',' to join each item in an array, and break an new line when reaching the width (default is 16)
#
# @param Array: The array need to be formatted
# @param Width: The line length, the default value is set to 16
#
# @retval ArrayItem: A string for all formatted array items
#
def CreateArrayItem(Array, Width = 16):
MaxLength = Width
Index = 0
Line = ' '
ArrayItem = []
for Item in Array:
if Index < MaxLength:
Line = Line + Item + ', '
Index = Index + 1
else:
ArrayItem = WriteLine(ArrayItem, Line)
Line = ' ' + Item + ', '
Index = 1
ArrayItem = Write(ArrayItem, Line.rstrip())
return "".join(ArrayItem)
## CreateCFileStringValue
#
# Create a line with string value
#
# @param Value: Value of the string
#
# @retval Str: A formatted string with string value
#
def CreateCFileStringValue(Value):
Value = [StringBlockType] + Value
Str = WriteLine('', CreateArrayItem(Value))
return "".join(Str)
## GetFilteredLanguage
#
# apply get best language rules to the UNI language code list
#
# @param UniLanguageList: language code definition list in *.UNI file
# @param LanguageFilterList: language code filter list of RFC4646 format in DSC file
#
# @retval UniLanguageListFiltered: the filtered language code
#
def GetFilteredLanguage(UniLanguageList, LanguageFilterList):
UniLanguageListFiltered = []
# if filter list is empty, then consider there is no filter
if LanguageFilterList == []:
UniLanguageListFiltered = UniLanguageList
return UniLanguageListFiltered
for Language in LanguageFilterList:
# first check for exact match
if Language in UniLanguageList:
if Language not in UniLanguageListFiltered:
UniLanguageListFiltered.append(Language)
# find the first one with the same/equivalent primary tag
else:
if Language.find('-') != -1:
PrimaryTag = Language[0:Language.find('-')].lower()
else:
PrimaryTag = Language
if len(PrimaryTag) == 3:
PrimaryTag = LangConvTable.get(PrimaryTag)
for UniLanguage in UniLanguageList:
if UniLanguage.find('-') != -1:
UniLanguagePrimaryTag = UniLanguage[0:UniLanguage.find('-')].lower()
else:
UniLanguagePrimaryTag = UniLanguage
if len(UniLanguagePrimaryTag) == 3:
UniLanguagePrimaryTag = LangConvTable.get(UniLanguagePrimaryTag)
if PrimaryTag == UniLanguagePrimaryTag:
if UniLanguage not in UniLanguageListFiltered:
UniLanguageListFiltered.append(UniLanguage)
break
else:
# Here is rule 3 for "get best language"
# If tag is not listed in the Unicode file, the default ("en") tag should be used for that language
# for better processing, find the one that best suit for it.
DefaultTag = 'en'
if DefaultTag not in UniLanguageListFiltered:
# check whether language code with primary code equivalent with DefaultTag already in the list, if so, use that
for UniLanguage in UniLanguageList:
if UniLanguage.startswith('en-') or UniLanguage.startswith('eng-'):
if UniLanguage not in UniLanguageListFiltered:
UniLanguageListFiltered.append(UniLanguage)
break
else:
UniLanguageListFiltered.append(DefaultTag)
return UniLanguageListFiltered
## Create content of .c file
#
# Create content of .c file
#
# @param BaseName: The basename of strings
# @param UniObjectClass A UniObjectClass instance
# @param IsCompatibleMode Compatible mode
# @param UniBinBuffer UniBinBuffer to contain UniBinary data.
# @param FilterInfo Platform language filter information
#
# @retval Str: A string of .c file content
#
def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniBinBuffer, FilterInfo):
#
# Init array length
#
TotalLength = EFI_HII_ARRAY_SIZE_LENGTH
Str = ''
Offset = 0
EDK2Module = FilterInfo[0]
if EDK2Module:
LanguageFilterList = FilterInfo[1]
else:
# EDK module is using ISO639-2 format filter, convert to the RFC4646 format
LanguageFilterList = [LangConvTable.get(F.lower()) for F in FilterInfo[1]]
UniLanguageList = []
for IndexI in range(len(UniObjectClass.LanguageDef)):
UniLanguageList += [UniObjectClass.LanguageDef[IndexI][0]]
UniLanguageListFiltered = GetFilteredLanguage(UniLanguageList, LanguageFilterList)
#
# Create lines for each language's strings
#
for IndexI in range(len(UniObjectClass.LanguageDef)):
Language = UniObjectClass.LanguageDef[IndexI][0]
if Language not in UniLanguageListFiltered:
continue
StringBuffer = BytesIO()
StrStringValue = ''
ArrayLength = 0
NumberOfUseOtherLangDef = 0
Index = 0
for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):
Item = UniObjectClass.OrderedStringListByToken[Language][IndexJ]
Name = Item.StringName
Value = Item.StringValueByteList
Referenced = Item.Referenced
Token = Item.Token
UseOtherLangDef = Item.UseOtherLangDef
if UseOtherLangDef != '' and Referenced:
NumberOfUseOtherLangDef = NumberOfUseOtherLangDef + 1
Index = Index + 1
else:
if NumberOfUseOtherLangDef > 0:
StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
CreateBinBuffer (StringBuffer, ([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
NumberOfUseOtherLangDef = 0
ArrayLength = ArrayLength + 3
if Referenced and Item.Token > 0:
Index = Index + 1
StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))
StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))
CreateBinBuffer (StringBuffer, [StringBlockType] + Value)
ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type
#
# EFI_HII_PACKAGE_HEADER
#
Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1
ArrayLength = Offset + ArrayLength + 1
#
# Create PACKAGE HEADER
#
Str = WriteLine(Str, '// PACKAGE HEADER\n')
TotalLength = TotalLength + ArrayLength
List = DecToHexList(ArrayLength, 6) + \
[StringPackageType] + \
DecToHexList(Offset) + \
DecToHexList(Offset) + \
DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \
DecToHexList(EFI_STRING_ID, 4) + \
AscToHexList(Language) + \
DecToHexList(0, 2)
Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n')
#
# Create PACKAGE DATA
#
Str = WriteLine(Str, '// PACKAGE DATA\n')
Str = Write(Str, StrStringValue)
#
# Add an EFI_HII_SIBT_END at last
#
Str = WriteLine(Str, ' ' + EFI_HII_SIBT_END + ",")
#
# Create binary UNI string
#
if UniBinBuffer:
CreateBinBuffer (UniBinBuffer, List)
UniBinBuffer.write (StringBuffer.getvalue())
UniBinBuffer.write (pack("B", int(EFI_HII_SIBT_END, 16)))
StringBuffer.close()
#
# Create line for string variable name
# "unsigned char $(BaseName)Strings[] = {"
#
AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n')
if IsCompatibleMode:
#
# Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode
#
AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')
AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')
else:
#
# Create whole array length in UEFI mode
#
AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')
#
# Join package data
#
AllStr = Write(AllStr, Str)
return "".join(AllStr)
## Create end of .c file
#
# Create end of .c file
#
# @retval Str: A string of .h file end
#
def CreateCFileEnd():
Str = Write('', '};')
return Str
## Create a .c file
#
# Create a complete .c file
#
# @param BaseName: The basename of strings
# @param UniObjectClass A UniObjectClass instance
# @param IsCompatibleMode Compatible Mode
# @param FilterInfo Platform language filter information
#
# @retval CFile: A string of complete .c file
#
def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode, FilterInfo):
CFile = ''
CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, None, FilterInfo))
CFile = WriteLine(CFile, CreateCFileEnd())
return "".join(CFile)
## GetFileList
#
# Get a list for all files
#
# @param IncludeList: A list of all path to be searched
# @param SkipList: A list of all types of file could be skipped
#
# @retval FileList: A list of all files found
#
def GetFileList(SourceFileList, IncludeList, SkipList):
if IncludeList is None:
EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined")
FileList = []
if SkipList is None:
SkipList = []
for File in SourceFileList:
for Dir in IncludeList:
if not os.path.exists(Dir):
continue
File = os.path.join(Dir, File.Path)
#
# Ignore Dir
#
if os.path.isfile(File) != True:
continue
#
# Ignore file listed in skip list
#
IsSkip = False
for Skip in SkipList:
if os.path.splitext(File)[1].upper() == Skip.upper():
EdkLogger.verbose("Skipped %s for string token uses search" % File)
IsSkip = True
break
if not IsSkip:
FileList.append(File)
break
return FileList
## SearchString
#
# Search whether all string defined in UniObjectClass are referenced
# All string used should be set to Referenced
#
# @param UniObjectClass: Input UniObjectClass
# @param FileList: Search path list
# @param IsCompatibleMode Compatible Mode
#
# @retval UniObjectClass: UniObjectClass after searched
#
def SearchString(UniObjectClass, FileList, IsCompatibleMode):
if FileList == []:
return UniObjectClass
for File in FileList:
try:
if os.path.isfile(File):
Lines = open(File, 'r')
for Line in Lines:
for StrName in STRING_TOKEN.findall(Line):
EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)
UniObjectClass.SetStringReferenced(StrName)
except:
EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "SearchString: Error while processing file", File=File, RaiseError=False)
raise
UniObjectClass.ReToken()
return UniObjectClass
## GetStringFiles
#
# This function is used for UEFI2.1 spec
#
#
def GetStringFiles(UniFilList, SourceFileList, IncludeList, IncludePathList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False, UniGenCFlag = True, UniGenBinBuffer = None, FilterInfo = [True, []]):
if len(UniFilList) > 0:
if ShellMode:
#
# support ISO 639-2 codes in .UNI files of EDK Shell
#
Uni = UniFileClassObject(sorted(UniFilList, key=lambda x: x.File), True, IncludePathList)
else:
Uni = UniFileClassObject(sorted(UniFilList, key=lambda x: x.File), IsCompatibleMode, IncludePathList)
else:
EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')
FileList = GetFileList(SourceFileList, IncludeList, SkipList)
Uni = SearchString(Uni, sorted (FileList), IsCompatibleMode)
HFile = CreateHFile(BaseName, Uni, IsCompatibleMode, UniGenCFlag)
CFile = None
if IsCompatibleMode or UniGenCFlag:
CFile = CreateCFile(BaseName, Uni, IsCompatibleMode, FilterInfo)
if UniGenBinBuffer:
CreateCFileContent(BaseName, Uni, IsCompatibleMode, UniGenBinBuffer, FilterInfo)
return HFile, CFile
#
# Write an item
#
def Write(Target, Item):
if isinstance(Target,str):
Target = [Target]
if not Target:
Target = []
if isinstance(Item,list):
Target.extend(Item)
else:
Target.append(Item)
return Target
#
# Write an item with a break line
#
def WriteLine(Target, Item):
if isinstance(Target,str):
Target = [Target]
if not Target:
Target = []
if isinstance(Item, list):
Target.extend(Item)
else:
Target.append(Item)
Target.append('\n')
return Target
# This acts like the main() function for the script, unless it is 'import'ed into another
# script.
if __name__ == '__main__':
EdkLogger.info('start')
UniFileList = [
r'C:\\Edk\\Strings2.uni',
r'C:\\Edk\\Strings.uni'
]
SrcFileList = []
for Root, Dirs, Files in os.walk('C:\\Edk'):
for File in Files:
SrcFileList.append(File)
IncludeList = [
r'C:\\Edk'
]
SkipList = ['.inf', '.uni']
BaseName = 'DriverSample'
(h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True)
hfile = open('unistring.h', 'w')
cfile = open('unistring.c', 'w')
hfile.write(h)
cfile.write(c)
EdkLogger.info('end')
Reference in New Issue View Git Blame Copy Permalink