BaseTools: Convert Split tool to python

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=3165

There are 2 reasons to convert Split tool from C to Python.
1. We are in the process of moving the Basetools Python code
to a separate repository. But there still are many C tools under
edk2/BaseTools. To make all Basetools be in the separate repo,
we can convert the C tools to Python tools.
2. The original Split tool is very slow. This python tool can reduce
90% time.

Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Yuwei Chen <yuwei.chen@intel.com>

Reviewed-by: Liming Gao <gaoliming@byosoft.com.cn>
Reviewed-by: Yuwei Chen <yuwei.chen@intel.com>
This commit is contained in:
Bob Feng 2021-01-13 11:12:24 +08:00 committed by mergify[bot]
parent 339371ef78
commit 5b4a97bbc3
9 changed files with 223 additions and 524 deletions

View File

@ -1,29 +1,14 @@
#!/usr/bin/env bash
#python `dirname $0`/RunToolFromSource.py `basename $0` $*
# If a ${PYTHON_COMMAND} command is available, use it in preference to python
if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
python_exe=${PYTHON_COMMAND}
fi
full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
dir=$(dirname "$full_cmd")
cmd=${full_cmd##*/}
if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]
then
exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"
elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]
then
if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]
then
echo "BaseTools C Tool binary was not found ($cmd)"
echo "You may need to run:"
echo " make -C $EDK_TOOLS_PATH/Source/C"
else
exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"
fi
elif [ -e "$dir/../../Source/C/bin/$cmd" ]
then
exec "$dir/../../Source/C/bin/$cmd" "$@"
else
echo "Unable to find the real '$cmd' to run"
echo "This message was printed by"
echo " $0"
exit 127
fi
export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
exec "${python_exe:-python}" "$dir/../../Source/Python/$cmd/$cmd.py" "$@"

View File

@ -0,0 +1,3 @@
@setlocal
@set ToolName=%~n0%
@%PYTHON_COMMAND% %BASE_TOOLS_PATH%\Source\Python\%ToolName%\%ToolName%.py %*

View File

@ -57,7 +57,6 @@ APPLICATIONS = \
GenSec \
GenCrc32 \
LzmaCompress \
Split \
TianoCompress \
VolInfo \
DevicePath

View File

@ -19,7 +19,6 @@ APPLICATIONS = \
GenFw \
GenSec \
LzmaCompress \
Split \
TianoCompress \
VolInfo \
DevicePath

View File

@ -1,17 +0,0 @@
## @file
# GNU/Linux makefile for 'Split' module build.
#
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
MAKEROOT ?= ..
APPNAME = Split
OBJECTS = Split.o
include $(MAKEROOT)/Makefiles/app.makefile
LIBS = -lCommon

View File

@ -1,16 +0,0 @@
## @file
# Windows makefile for 'Split' module build.
#
# Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
!INCLUDE ..\Makefiles\ms.common
APPNAME = Split
LIBS = $(LIB_PATH)\Common.lib
OBJECTS = Split.obj
!INCLUDE ..\Makefiles\ms.app

View File

@ -1,466 +0,0 @@
/** @file
Split a file into two pieces at the request offset.
Copyright (c) 1999 - 2017, Intel Corporation. All rights reserved.<BR>
SPDX-License-Identifier: BSD-2-Clause-Patent
**/
// GC_TODO: fix comment to start with /*++
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef __GNUC__
#include <unistd.h>
#else
#include <direct.h>
#endif
#include <ctype.h>
#include "ParseInf.h"
#include "CommonLib.h"
#include "EfiUtilityMsgs.h"
//
// Utility Name
//
#define UTILITY_NAME "Split"
//
// Utility version information
//
#define UTILITY_MAJOR_VERSION 1
#define UTILITY_MINOR_VERSION 0
void
Version (
void
)
/*++
Routine Description:
Displays the standard utility information to SDTOUT
Arguments:
None
Returns:
None
--*/
{
printf ("%s Version %d.%d Build %s\n", UTILITY_NAME, UTILITY_MAJOR_VERSION, UTILITY_MINOR_VERSION, __BUILD_VERSION);
}
void
Usage (
void
)
/*++
Routine Description:
GC_TODO: Add function description
Arguments:
Returns:
GC_TODO: add return values
--*/
{
Version();
printf ("Copyright (c) 1999-2017 Intel Corporation. All rights reserved.\n");
printf ("\n SplitFile creates two Binary files either in the same directory as the current working\n");
printf (" directory or in the specified directory.\n");
printf ("\nUsage: \n\
Split\n\
-f, --filename inputFile to split\n\
-s, --split VALUE the number of bytes in the first file\n\
[-p, --prefix OutputDir]\n\
[-o, --firstfile Filename1]\n\
[-t, --secondfile Filename2]\n\
[-v, --verbose]\n\
[--version]\n\
[-q, --quiet disable all messages except fatal errors]\n\
[-d, --debug[#]\n\
[-h, --help]\n");
}
EFI_STATUS
GetSplitValue (
IN CONST CHAR8* SplitValueString,
OUT UINT64 *ReturnValue
)
{
UINT64 len = 0;
UINT64 base = 1;
UINT64 index = 0;
UINT64 number = 0;
CHAR8 lastCHAR = 0;
EFI_STATUS Status = EFI_SUCCESS;
if (SplitValueString != NULL){
len = strlen(SplitValueString);
}
if (len == 0) {
return EFI_ABORTED;
}
Status = AsciiStringToUint64 (SplitValueString, FALSE, ReturnValue);
if (!EFI_ERROR (Status)) {
return Status;
}
if (SplitValueString[0] == '0' && (SplitValueString[1] == 'x' || SplitValueString[1] == 'X')) {
Status = AsciiStringToUint64 (SplitValueString, TRUE, ReturnValue);
if (!EFI_ERROR (Status)) {
return Status;
}
}
lastCHAR = (CHAR8)toupper((int)SplitValueString[len - 1]);
if (lastCHAR != 'K' && lastCHAR != 'M' && lastCHAR != 'G') {
return STATUS_ERROR;
}
for (;index < len - 1; ++index) {
if (!isdigit((int)SplitValueString[index])) {
return EFI_ABORTED;
}
}
number = atol (SplitValueString);
if (lastCHAR == 'K')
base = 1024;
else if (lastCHAR == 'M')
base = 1024*1024;
else
base = 1024*1024*1024;
*ReturnValue = number*base;
return EFI_SUCCESS;
}
EFI_STATUS
CountVerboseLevel (
IN CONST CHAR8* VerboseLevelString,
IN CONST UINT64 Length,
OUT UINT64 *ReturnValue
)
{
UINT64 i = 0;
for (;i < Length; ++i) {
if (VerboseLevelString[i] != 'v' && VerboseLevelString[i] != 'V') {
return EFI_ABORTED;
}
++(*ReturnValue);
}
return EFI_SUCCESS;
}
EFI_STATUS
CreateDir (
IN OUT CHAR8** FullFileName
)
{
CHAR8* temp = *FullFileName;
CHAR8* start = temp;
CHAR8 tempchar;
UINT64 index = 0;
for (;index < strlen(temp); ++index) {
if (temp[index] == '\\' || temp[index] == '/') {
if (temp[index + 1] != '\0') {
tempchar = temp[index + 1];
temp[index + 1] = 0;
if (chdir(start)) {
if (mkdir(start, S_IRWXU | S_IRWXG | S_IRWXO) != 0) {
return EFI_ABORTED;
}
chdir(start);
}
start = temp + index + 1;
temp[index] = '/';
temp[index + 1] = tempchar;
}
}
}
return EFI_SUCCESS;
}
int
main (
int argc,
char*argv[]
)
/*++
Routine Description:
GC_TODO: Add function description
Arguments:
argc - GC_TODO: add argument description
] - GC_TODO: add argument description
Returns:
GC_TODO: add return values
--*/
{
EFI_STATUS Status = EFI_SUCCESS;
INTN ReturnStatus = STATUS_SUCCESS;
FILE *In;
CHAR8 *InputFileName = NULL;
CHAR8 *OutputDir = NULL;
CHAR8 *OutFileName1 = NULL;
CHAR8 *OutFileName2 = NULL;
UINT64 SplitValue = (UINT64) -1;
FILE *Out1 = NULL;
FILE *Out2 = NULL;
CHAR8 *OutName1 = NULL;
CHAR8 *OutName2 = NULL;
CHAR8 *CurrentDir = NULL;
UINT64 Index;
CHAR8 CharC;
UINT64 DebugLevel = 0;
UINT64 VerboseLevel = 0;
SetUtilityName(UTILITY_NAME);
if (argc == 1) {
Usage();
return STATUS_ERROR;
}
argc --;
argv ++;
if ((stricmp (argv[0], "-h") == 0) || (stricmp (argv[0], "--help") == 0)) {
Usage();
return STATUS_SUCCESS;
}
if (stricmp (argv[0], "--version") == 0) {
Version();
return STATUS_SUCCESS;
}
while (argc > 0) {
if ((stricmp (argv[0], "-p") == 0) || (stricmp (argv[0], "--prefix") == 0)) {
OutputDir = argv[1];
if (OutputDir == NULL) {
Warning (NULL, 0, 0, "NO output directory specified.", NULL);
return STATUS_ERROR;
}
argc -= 2;
argv += 2;
continue;
}
if ((stricmp (argv[0], "-f") == 0) || (stricmp (argv[0], "--filename") == 0)) {
InputFileName = argv[1];
if (InputFileName == NULL) {
Error (NULL, 0, 0x1001, "NO Input file specified.", NULL);
return STATUS_ERROR;
}
argc -= 2;
argv += 2;
continue;
}
if ((stricmp (argv[0], "-s") == 0) || (stricmp (argv[0], "--split") == 0)) {
Status = GetSplitValue(argv[1], &SplitValue);
if (EFI_ERROR (Status)) {
Error (NULL, 0, 0x1003, "Input split value is not one valid integer.", NULL);
return STATUS_ERROR;
}
argc -= 2;
argv += 2;
continue;
}
if ((stricmp (argv[0], "-o") == 0) || (stricmp (argv[0], "--firstfile") == 0)) {
OutFileName1 = argv[1];
if (OutFileName1 == NULL) {
Warning (NULL, 0, 0, NULL, "No output file1 specified.");
}
argc -= 2;
argv += 2;
continue;
}
if ((stricmp (argv[0], "-t") == 0) || (stricmp (argv[0], "--secondfile") == 0)) {
OutFileName2 = argv[1];
if (OutFileName2 == NULL) {
Warning (NULL, 0, 0, NULL, "No output file2 specified.");
}
argc -= 2;
argv += 2;
continue;
}
if ((stricmp (argv[0], "-q") == 0) || (stricmp (argv[0], "--quiet") == 0)) {
argc --;
argv ++;
continue;
}
if ((strlen(argv[0]) >= 2 && argv[0][0] == '-' && (argv[0][1] == 'v' || argv[0][1] == 'V')) || (stricmp (argv[0], "--verbose") == 0)) {
VerboseLevel = 1;
if (strlen(argv[0]) > 2) {
Status = CountVerboseLevel (&argv[0][2], strlen(argv[0]) - 2, &VerboseLevel);
if (EFI_ERROR (Status)) {
Error (NULL, 0, 0x1003, NULL, "%s is invalid parameter!", argv[0]);
return STATUS_ERROR;
}
}
argc --;
argv ++;
continue;
}
if ((stricmp (argv[0], "-d") == 0) || (stricmp (argv[0], "--debug") == 0)) {
Status = AsciiStringToUint64 (argv[1], FALSE, &DebugLevel);
if (EFI_ERROR (Status)) {
Error (NULL, 0, 0x1003, "Input debug level is not one valid integrator.", NULL);
return STATUS_ERROR;
}
argc -= 2;
argv += 2;
continue;
}
//
// Don't recognize the parameter.
//
Error (NULL, 0, 0x1003, NULL, "%s is invalid parameter!", argv[0]);
return STATUS_ERROR;
}
if (InputFileName == NULL) {
Error (NULL, 0, 0x1001, "NO Input file specified.", NULL);
return STATUS_ERROR;
}
In = fopen (LongFilePath (InputFileName), "rb");
if (In == NULL) {
// ("Unable to open file \"%s\"\n", InputFileName);
Error (InputFileName, 0, 1, "File open failure", NULL);
return STATUS_ERROR;
}
if (OutFileName1 == NULL) {
OutName1 = (CHAR8*)malloc(strlen(InputFileName) + 16);
if (OutName1 == NULL) {
Warning (NULL, 0, 0, NULL, "Memory Allocation Fail.");
ReturnStatus = STATUS_ERROR;
goto Finish;
}
strcpy (OutName1, InputFileName);
strcat (OutName1, "1");
OutFileName1 = OutName1;
}
if (OutFileName2 == NULL) {
OutName2 = (CHAR8*)malloc(strlen(InputFileName) + 16);
if (OutName2 == NULL) {
Warning (NULL, 0, 0, NULL, "Memory Allocation Fail.");
ReturnStatus = STATUS_ERROR;
goto Finish;
}
strcpy (OutName2, InputFileName);
strcat (OutName2, "2");
OutFileName2 = OutName2;
}
if (OutputDir != NULL) {
//OutputDirSpecified = TRUE;
if (chdir(OutputDir) != 0) {
Warning (NULL, 0, 0, NULL, "Change dir to OutputDir Fail.");
ReturnStatus = STATUS_ERROR;
goto Finish;
}
}
CurrentDir = (CHAR8*)getcwd((CHAR8*)0, 0);
if (EFI_ERROR(CreateDir(&OutFileName1))) {
Error (OutFileName1, 0, 5, "Create Dir for File1 Fail.", NULL);
ReturnStatus = STATUS_ERROR;
goto Finish;
}
chdir(CurrentDir);
if (EFI_ERROR(CreateDir(&OutFileName2))) {
Error (OutFileName2, 0, 5, "Create Dir for File2 Fail.", NULL);
ReturnStatus = STATUS_ERROR;
goto Finish;
}
chdir(CurrentDir);
free(CurrentDir);
Out1 = fopen (LongFilePath (OutFileName1), "wb");
if (Out1 == NULL) {
// ("Unable to open file \"%s\"\n", OutFileName1);
Error (OutFileName1, 0, 1, "File open failure", NULL);
ReturnStatus = STATUS_ERROR;
goto Finish;
}
Out2 = fopen (LongFilePath (OutFileName2), "wb");
if (Out2 == NULL) {
// ("Unable to open file \"%s\"\n", OutFileName2);
Error (OutFileName2, 0, 1, "File open failure", NULL);
ReturnStatus = STATUS_ERROR;
goto Finish;
}
for (Index = 0; Index < SplitValue; Index++) {
CharC = (CHAR8) fgetc (In);
if (feof (In)) {
break;
}
fputc (CharC, Out1);
}
for (;;) {
CharC = (CHAR8) fgetc (In);
if (feof (In)) {
break;
}
fputc (CharC, Out2);
}
Finish:
if (OutName1 != NULL) {
free(OutName1);
}
if (OutName2 != NULL) {
free(OutName2);
}
if (In != NULL) {
fclose (In);
}
if (Out1 != NULL) {
fclose (Out1);
}
if (Out2 != NULL) {
fclose (Out2);
}
return ReturnStatus;
}

View File

@ -0,0 +1,202 @@
# @file
# Split a file into two pieces at the request offset.
#
# Copyright (c) 2021, Intel Corporation. All rights reserved.<BR>
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
##
# Import Modules
#
import argparse
import os
import io
import shutil
import logging
import sys
import tempfile
parser = argparse.ArgumentParser(description='''
SplitFile creates two Binary files either in the same directory as the current working directory or in the specified directory.
''')
parser.add_argument("-f", "--filename", dest="inputfile",
required=True, help="The input file to split tool.")
parser.add_argument("-s", "--split", dest="position",
required=True, help="The number of bytes in the first file. The valid format are HEX, Decimal and Decimal[KMG].")
parser.add_argument("-p", "--prefix", dest="output",
help="The output folder.")
parser.add_argument("-o", "--firstfile", help="The first file name")
parser.add_argument("-t", "--secondfile", help="The second file name")
parser.add_argument("--version", action="version", version='%(prog)s Version 2.0',
help="Print debug information.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-v", "--verbose", action="store_true",
help="Print debug information.")
group.add_argument("-q", "--quiet", action="store_true",
help="Disable all messages except fatal errors")
SizeDict = {
"K": 1024,
"M": 1024*1024,
"G": 1024*1024*1024
}
def GetPositionValue(position):
'''
Parse the string of the argument position and return a decimal number.
The valid position formats are
1. HEX
e.g. 0x1000 or 0X1000
2. Decimal
e.g. 100
3. Decimal[KMG]
e.g. 100K or 100M or 100G or 100k or 100m or 100g
'''
logger = logging.getLogger('Split')
PosVal = 0
header = position[:2].upper()
tailer = position[-1].upper()
try:
if tailer in SizeDict:
PosVal = int(position[:-1]) * SizeDict[tailer]
else:
if header == "0X":
PosVal = int(position, 16)
else:
PosVal = int(position)
except Exception as e:
logger.error(
"The parameter %s format is incorrect. The valid format is HEX, Decimal and Decimal[KMG]." % position)
raise(e)
return PosVal
def getFileSize(filename):
'''
Read the input file and return the file size.
'''
logger = logging.getLogger('Split')
length = 0
try:
with open(filename, "rb") as fin:
fin.seek(0, io.SEEK_END)
length = fin.tell()
except Exception as e:
logger.error("Access file failed: %s", filename)
raise(e)
return length
def splitFile(inputfile, position, outputdir=None, outputfile1=None, outputfile2=None):
'''
Split the inputfile into outputfile1 and outputfile2 from the position.
'''
logger = logging.getLogger('Split')
inputfile = os.path.abspath(inputfile)
workspace = os.path.dirname(inputfile)
if not os.path.exists(inputfile):
logger.error("File Not Found: %s" % inputfile)
raise(Exception)
if outputfile1 and outputfile2 and outputfile1 == outputfile2:
logger.error(
"The firstfile and the secondfile can't be the same: %s" % outputfile1)
raise(Exception)
if not outputdir:
outputdir = workspace
elif not os.path.isabs(outputdir):
outputdir = os.path.join(workspace, outputdir)
# Create dir for the output files
try:
if not outputfile1:
outputfile1 = os.path.abspath(os.path.join(
outputdir, "{}1".format(os.path.basename(inputfile))))
else:
outputfile1 = os.path.abspath(os.path.join(outputdir, outputfile1))
outputdir = os.path.dirname(outputfile1)
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if not outputfile2:
outputfile2 = os.path.abspath(os.path.join(
outputdir, "{}2".format(os.path.basename(inputfile))))
else:
outputfile2 = os.path.abspath(os.path.join(outputdir, outputfile2))
outputdir = os.path.dirname(outputfile2)
if not os.path.exists(outputdir):
os.makedirs(outputdir)
except Exception as e:
logger.error("Can't make dir: %s" % outputdir)
raise(e)
if position <= 0:
if outputfile2 != inputfile:
shutil.copy2(inputfile, outputfile2)
with open(outputfile1, "wb") as fout:
fout.write(b'')
else:
inputfilesize = getFileSize(inputfile)
if position >= inputfilesize:
if outputfile1 != inputfile:
shutil.copy2(inputfile, outputfile1)
with open(outputfile2, "wb") as fout:
fout.write(b'')
else:
try:
tempdir = tempfile.mkdtemp()
tempfile1 = os.path.join(tempdir, "file1.bin")
tempfile2 = os.path.join(tempdir, "file2.bin")
with open(inputfile, "rb") as fin:
content1 = fin.read(position)
with open(tempfile1, "wb") as fout1:
fout1.write(content1)
content2 = fin.read(inputfilesize - position)
with open(tempfile2, "wb") as fout2:
fout2.write(content2)
shutil.copy2(tempfile1, outputfile1)
shutil.copy2(tempfile2, outputfile2)
except Exception as e:
logger.error("Split file failed")
raise(e)
finally:
if os.path.exists(tempdir):
shutil.rmtree(tempdir)
def main():
args = parser.parse_args()
status = 0
logger = logging.getLogger('Split')
if args.quiet:
logger.setLevel(logging.CRITICAL)
if args.verbose:
logger.setLevel(logging.DEBUG)
lh = logging.StreamHandler(sys.stdout)
lf = logging.Formatter("%(levelname)-8s: %(message)s")
lh.setFormatter(lf)
logger.addHandler(lh)
try:
position = GetPositionValue(args.position)
splitFile(args.inputfile, position, args.output,
args.firstfile, args.secondfile)
except Exception as e:
status = 1
return status
if __name__ == "__main__":
exit(main())

View File

@ -0,0 +1,10 @@
# @file
# Split a file into two pieces at the request offset.
#
# Copyright (c) 2021, Intel Corporation. All rights reserved.<BR>
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
##
# Import Modules