.pytool/Plugin: Add CI plugins

https://bugzilla.tianocore.org/show_bug.cgi?id=2315

Add .pytool directory to the edk2 repository with the
following plugins.  These plugins are in a top level
directory because that can be used with all packages
and platforms.

* CharEncodingCheck
* CompilerPlugin
* DependencyCheck
* DscCompleteCheck
* GuidCheck
* LibraryClassCheck
* SpellCheck

Cc: Sean Brogan <sean.brogan@microsoft.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Sean Brogan 2019-10-17 21:40:58 -07:00 committed by Michael D Kinney
parent de4ce46d6e
commit 9da7846c88
22 changed files with 1648 additions and 0 deletions

View File

@ -0,0 +1,118 @@
# @file CharEncodingCheck.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import os
import logging
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toolext.environment.var_dict import VarDict
##
# map
##
EcodingMap = {
".md": 'utf-8',
".dsc": 'utf-8',
".dec": 'utf-8',
".c": 'utf-8',
".h": 'utf-8',
".asm": 'utf-8',
".masm": 'utf-8',
".nasm": 'utf-8',
".s": 'utf-8',
".inf": 'utf-8',
".asl": 'utf-8',
".uni": 'utf-8',
".py": 'utf-8'
}
class CharEncodingCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that scans each file in the code tree and confirms the encoding is correct.
Configuration options:
"CharEncodingCheck": {
"IgnoreFiles": []
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
"""
return ("Check for valid file encoding for " + packagename, packagename + ".CharEncodingCheck")
##
# External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
overall_status = 0
files_tested = 0
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
if abs_pkg_path is None:
tc.SetSkipped()
tc.LogStdError("No Package folder {0}".format(abs_pkg_path))
return 0
for (ext, enc) in EcodingMap.items():
files = self.WalkDirectoryForExtension([ext], abs_pkg_path)
files = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in files] # make edk2relative path so can process ignores
if "IgnoreFiles" in pkgconfig:
for a in pkgconfig["IgnoreFiles"]:
a = a.replace(os.sep, "/")
try:
tc.LogStdOut("Ignoring File {0}".format(a))
files.remove(a)
except:
tc.LogStdError("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
logging.info("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
files = [Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(x) for x in files]
for a in files:
files_tested += 1
if(self.TestEncodingOk(a, enc)):
logging.debug("File {0} Passed Encoding Check {1}".format(a, enc))
else:
tc.LogStdError("Encoding Failure in {0}. Not {1}".format(a, enc))
overall_status += 1
tc.LogStdOut("Tested Encoding on {0} files".format(files_tested))
if overall_status is not 0:
tc.SetFailed("CharEncoding {0} Failed. Errors {1}".format(packagename, overall_status), "CHAR_ENCODING_CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status
def TestEncodingOk(self, apath, encodingValue):
try:
with open(apath, "rb") as fobj:
fobj.read().decode(encodingValue)
except Exception as exp:
logging.error("Encoding failure: file: {0} type: {1}".format(apath, encodingValue))
logging.debug("EXCEPTION: while processing {1} - {0}".format(exp, apath))
return False
return True

View File

@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to check char encoding
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Char Encoding Check Test",
"module": "CharEncodingCheck"
}

View File

@ -0,0 +1,18 @@
# Character Encoding Check Plugin
This CiBuildPlugin scans all the files in a package to make sure each file is
correctly encoded and all characters can be read. Improper encoding causes
tools to fail in some situations especially in different locals.
## Configuration
The plugin can be configured to ignore certain files.
``` yaml
"CharEncodingCheck": {
"IgnoreFiles": []
}
```
### IgnoreFiles
OPTIONAL List of file to ignore.

View File

@ -0,0 +1,102 @@
# @file HostUnitTestCompiler_plugin.py
##
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import os
import re
from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toolext.environment.uefi_build import UefiBuilder
from edk2toolext import edk2_logging
from edk2toolext.environment.var_dict import VarDict
class CompilerPlugin(ICiBuildPlugin):
"""
A CiBuildPlugin that compiles the package dsc
from the package being tested.
Configuration options:
"CompilerPlugin": {
"DscPath": "<path to dsc from root of pkg>"
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
"""
target = environment.GetValue("TARGET")
return ("Compile " + packagename + " " + target, packagename + ".Compiler." + target)
def RunsOnTargetList(self):
return ["DEBUG", "RELEASE"]
##
# External function of plugin. This function is used to perform the task of the MuBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
self._env = environment
# Parse the config for required DscPath element
if "DscPath" not in pkgconfig:
tc.SetSkipped()
tc.LogStdError("DscPath not found in config file. Nothing to compile.")
return -1
AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
APDSC = os.path.join(AP, pkgconfig["DscPath"].strip())
AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC)
if AP is None or AP_Path is None or not os.path.isfile(APDSC):
tc.SetSkipped()
tc.LogStdError("Package Dsc not found.")
return -1
logging.info("Building {0}".format(AP_Path))
self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin")
# Parse DSC to check for SUPPORTED_ARCHITECTURES
dp = DscParser()
dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
dp.SetPackagePaths(Edk2pathObj.PackagePathList)
dp.ParseFile(AP_Path)
if "SUPPORTED_ARCHITECTURES" in dp.LocalVars:
SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|')
TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ')
# Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES
if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0:
tc.SetSkipped()
tc.LogStdError("No supported architecutres to build")
return -1
uefiBuilder = UefiBuilder()
# do all the steps
# WorkSpace, PackagesPath, PInHelper, PInManager
ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM)
if ret != 0: # failure:
tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED")
tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret))
return 1
else:
tc.SetSuccess()
return 0

View File

@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to compile each package
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Compiler Plugin",
"module": "CompilerPlugin"
}

View File

@ -0,0 +1,17 @@
# Compiler Plugin
This CiBuildPlugin compiles the package DSC from the package being tested.
## Configuration
The package relative path of the DSC file to build.
``` yaml
"CompilerPlugin": {
"DscPath": "<path to dsc from root of pkg>"
}
```
### DscPath
Package relative path to the DSC file to build.

View File

@ -0,0 +1,120 @@
# @file dependency_check.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import os
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toolext.environment.var_dict import VarDict
class DependencyCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that finds all modules (inf files) in a package and reviews the packages used
to confirm they are acceptable. This is to help enforce layering and identify improper
dependencies between packages.
Configuration options:
"DependencyCheck": {
"AcceptableDependencies": [], # Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec
"AcceptableDependencies-<MODULE_TYPE>": [], # OPTIONAL Package dependencies for INFs that are HOST_APPLICATION
"AcceptableDependencies-HOST_APPLICATION": [], # EXAMPLE Package dependencies for INFs that are HOST_APPLICATION
"IgnoreInf": [] # Ignore INF if found in filesystem
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
"""
return ("Test Package Dependencies for modules in " + packagename, packagename + ".DependencyCheck")
##
# External function of plugin. This function is used to perform the task of the MuBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
overall_status = 0
# Get current platform
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
# Get INF Files
INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with Ignore List
# Remove ignored INFs
if "IgnoreInf" in pkgconfig:
for a in pkgconfig["IgnoreInf"]:
a = a.replace(os.sep, "/") ## convert path sep in case ignore list is bad. Can't change case
try:
INFFiles.remove(a)
tc.LogStdOut("IgnoreInf {0}".format(a))
except:
logging.info("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
tc.LogStdError("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
# Get the AccpetableDependencies list
if "AcceptableDependencies" not in pkgconfig:
logging.info("DependencyCheck Skipped. No Acceptable Dependencies defined.")
tc.LogStdOut("DependencyCheck Skipped. No Acceptable Dependencies defined.")
tc.SetSkipped()
return -1
# Log dependencies
for k in pkgconfig.keys():
if k.startswith("AcceptableDependencies"):
pkgstring = "\n".join(pkgconfig[k])
if ("-" in k):
_, _, mod_type = k.partition("-")
tc.LogStdOut(f"Additional dependencies for MODULE_TYPE {mod_type}:\n {pkgstring}")
else:
tc.LogStdOut(f"Acceptable Dependencies:\n {pkgstring}")
# For each INF file
for file in INFFiles:
ip = InfParser()
logging.debug("Parsing " + file)
ip.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList).ParseFile(file)
if("MODULE_TYPE" not in ip.Dict):
tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(file))
continue
mod_type = ip.Dict["MODULE_TYPE"].upper()
for p in ip.PackagesUsed:
if p not in pkgconfig["AcceptableDependencies"]:
# If not in the main acceptable dependencies list then check module specific
mod_specific_key = "AcceptableDependencies-" + mod_type
if mod_specific_key in pkgconfig and p in pkgconfig[mod_specific_key]:
continue
logging.error("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p))
tc.LogStdError("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p))
overall_status += 1
# If XML object exists, add results
if overall_status is not 0:
tc.SetFailed("Failed with {0} errors".format(overall_status), "DEPENDENCYCHECK_FAILED")
else:
tc.SetSuccess()
return overall_status

View File

@ -0,0 +1,13 @@
## @file
# CiBuildPlugin used to check all infs within a package
# to confirm the packagesdependency are on the configured list of acceptable
# dependencies.
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Dependency Check Test",
"module": "DependencyCheck"
}

View File

@ -0,0 +1,31 @@
# Depdendency Check Plugin
A CiBuildPlugin that finds all modules (inf files) in a package and reviews the
packages used to confirm they are acceptable. This is to help enforce layering
and identify improper dependencies between packages.
## Configuration
The plugin must be configured with the acceptabe package dependencies for the
package.
``` yaml
"DependencyCheck": {
"AcceptableDependencies": [],
"AcceptableDependencies-<MODULE_TYPE>": [],
"IgnoreInf": []
}
```
### AcceptableDependencies
Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec
### AcceptableDependencies-<MODULE_TYPE>
OPTIONAL Package dependencies for INFs that have module type <MODULE_TYPE>.
Example: AcceptableDependencies-HOST_APPLICATION.
### IgnoreInf
OPTIONAL list of INFs to ignore for this dependency check.

View File

@ -0,0 +1,118 @@
# @file DscCompleteCheck.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import os
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toolext.environment.var_dict import VarDict
class DscCompleteCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that scans the package dsc file and confirms all modules (inf files) are
listed in the components sections.
Configuration options:
"DscCompleteCheck": {
"DscPath": "<path to dsc from root of pkg>"
"IgnoreInf": [] # Ignore INF if found in filesystem by not dsc
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
"""
return ("Check the " + packagename + " DSC for a being complete", packagename + ".DscCompleteCheck")
##
# External function of plugin. This function is used to perform the task of the MuBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - VarDict containing the shell environment Build Vars
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
overall_status = 0
# Parse the config for required DscPath element
if "DscPath" not in pkgconfig:
tc.SetSkipped()
tc.LogStdError("DscPath not found in config file. Nothing to check.")
return -1
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip())
wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dsc_path)
if abs_dsc_path is None or wsr_dsc_path is "" or not os.path.isfile(abs_dsc_path):
tc.SetSkipped()
tc.LogStdError("Package Dsc not found")
return 0
# Get INF Files
INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with DSC
# remove ignores
if "IgnoreInf" in pkgconfig:
for a in pkgconfig["IgnoreInf"]:
a = a.replace(os.sep, "/")
try:
tc.LogStdOut("Ignoring INF {0}".format(a))
INFFiles.remove(a)
except:
tc.LogStdError("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
logging.info("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
# DSC Parser
dp = DscParser()
dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
dp.SetPackagePaths(Edk2pathObj.PackagePathList)
dp.SetInputVars(environment.GetAllBuildKeyValues())
dp.ParseFile(wsr_dsc_path)
# Check if INF in component section
for INF in INFFiles:
if not any(INF.strip() in x for x in dp.ThreeMods) and \
not any(INF.strip() in x for x in dp.SixMods) and \
not any(INF.strip() in x for x in dp.OtherMods):
infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath)
infp.SetPackagePaths(Edk2pathObj.PackagePathList)
infp.ParseFile(INF)
if("MODULE_TYPE" not in infp.Dict):
tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(INF))
continue
if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"):
tc.LogStdOut("Ignoring INF. Module type is HOST_APPLICATION {0}".format(INF))
continue
logging.critical(INF + " not in " + wsr_dsc_path)
tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path))
overall_status = overall_status + 1
# If XML object exists, add result
if overall_status is not 0:
tc.SetFailed("DscCompleteCheck {0} Failed. Errors {1}".format(wsr_dsc_path, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status

View File

@ -0,0 +1,12 @@
## @file
# CiBuildPlugin used to confirm all INFs are listed in
# the components section of package dsc
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Dsc Complete Check Test",
"module": "DscCompleteCheck"
}

View File

@ -0,0 +1,27 @@
# Dsc Complete Check Plugin
This CiBuildPlugin scans all INF files from a package and confirms they are
listed in the package level DSC file. The test considers it an error if any INF
does not appear in the `Components` section of the package-level DSC (indicating
that it would not be built if the package were built). This is critical because
much of the CI infrastructure assumes that all modules will be listed in the DSC
and compiled.
## Configuration
The plugin has a few configuration options to support the UEFI codebase.
``` yaml
"DscCompleteCheck": {
"DscPath": "", # Path to dsc from root of package
"IgnoreInf": [] # Ignore INF if found in filesystem by not dsc
}
```
### DscPath
Path to DSC to consider platform dsc
### IgnoreInf
Ignore error if Inf file is not listed in DSC file

View File

@ -0,0 +1,251 @@
# @file GuidCheck.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.uefi.edk2.guid_list import GuidList
from edk2toolext.environment.var_dict import VarDict
class GuidCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that scans the code tree and looks for duplicate guids
from the package being tested.
Configuration options:
"GuidCheck": {
"IgnoreGuidName": [], # provide in format guidname=guidvalue or just guidname
"IgnoreGuidValue": [],
"IgnoreFoldersAndFiles": [],
"IgnoreDuplicates": [] # Provide in format guidname=guidname=guidname...
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
"""
return ("Confirm GUIDs are unique in " + packagename, packagename + ".GuidCheck")
def _FindConflictingGuidValues(self, guidlist: list) -> list:
""" Find all duplicate guids by guid value and report them as errors
"""
# Sort the list by guid
guidsorted = sorted(
guidlist, key=lambda x: x.guid.upper(), reverse=True)
previous = None # Store previous entry for comparison
error = None
errors = []
for index in range(len(guidsorted)):
i = guidsorted[index]
if(previous is not None):
if i.guid == previous.guid: # Error
if(error is None):
# Catch errors with more than 1 conflict
error = ErrorEntry("guid")
error.entries.append(previous)
errors.append(error)
error.entries.append(i)
else:
# no match. clear error
error = None
previous = i
return errors
def _FindConflictingGuidNames(self, guidlist: list) -> list:
""" Find all duplicate guids by name and if they are not all
from inf files report them as errors. It is ok to have
BASE_NAME duplication.
Is this useful? It would catch two same named guids in dec file
that resolve to different values.
"""
# Sort the list by guid
namesorted = sorted(guidlist, key=lambda x: x.name.upper())
previous = None # Store previous entry for comparison
error = None
errors = []
for index in range(len(namesorted)):
i = namesorted[index]
if(previous is not None):
# If name matches
if i.name == previous.name:
if(error is None):
# Catch errors with more than 1 conflict
error = ErrorEntry("name")
error.entries.append(previous)
errors.append(error)
error.entries.append(i)
else:
# no match. clear error
error = None
previous = i
# Loop thru and remove any errors where all files are infs as it is ok if
# they have the same inf base name.
for e in errors[:]:
if len( [en for en in e.entries if not en.absfilepath.lower().endswith(".inf")]) == 0:
errors.remove(e)
return errors
##
# External function of plugin. This function is used to perform the task of the MuBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
Errors = []
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
packagename)
if abs_pkg_path is None:
tc.SetSkipped()
tc.LogStdError("No package {0}".format(packagename))
return -1
All_Ignores = ["/Build", "/Conf"]
# Parse the config for other ignores
if "IgnoreFoldersAndFiles" in pkgconfig:
All_Ignores.extend(pkgconfig["IgnoreFoldersAndFiles"])
# Parse the workspace for all GUIDs
gs = GuidList.guidlist_from_filesystem(
Edk2pathObj.WorkspacePath, ignore_lines=All_Ignores)
# Remove ignored guidvalue
if "IgnoreGuidValue" in pkgconfig:
for a in pkgconfig["IgnoreGuidValue"]:
try:
tc.LogStdOut("Ignoring Guid {0}".format(a.upper()))
for b in gs[:]:
if b.guid == a.upper():
gs.remove(b)
except:
tc.LogStdError("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper()))
logging.info("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper()))
# Remove ignored guidname
if "IgnoreGuidName" in pkgconfig:
for a in pkgconfig["IgnoreGuidName"]:
entry = a.split("=")
if(len(entry) > 2):
tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a))
logging.info("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a))
continue
try:
tc.LogStdOut("Ignoring Guid {0}".format(a))
for b in gs[:]:
if b.name == entry[0]:
if(len(entry) == 1):
gs.remove(b)
elif(len(entry) == 2 and b.guid.upper() == entry[1].upper()):
gs.remove(b)
else:
c.LogStdError("GuidCheck.IgnoreGuidName -> {0} incomplete match. Invalid ignore guid".format(a))
except:
tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a))
logging.info("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a))
# Find conflicting Guid Values
Errors.extend(self._FindConflictingGuidValues(gs))
# Check if there are expected duplicates and remove it from the error list
if "IgnoreDuplicates" in pkgconfig:
for a in pkgconfig["IgnoreDuplicates"]:
names = a.split("=")
if len(names) < 2:
tc.LogStdError("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a))
logging.info("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a))
continue
for b in Errors[:]:
if b.type != "guid":
continue
## Make a list of the names that are not in the names list. If there
## are any in the list then this error should not be ignored.
t = [x for x in b.entries if x.name not in names]
if(len(t) == len(b.entries)):
## did not apply to any entry
continue
elif(len(t) == 0):
## full match - ignore duplicate
tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0}".format(a))
Errors.remove(b)
elif(len(t) < len(b.entries)):
## partial match
tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a))
logging.info("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a))
else:
tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} unknown error.".format(a))
logging.info("GuidCheck.IgnoreDuplicates -> {0} unknown error".format(a))
# Find conflicting Guid Names
Errors.extend(self._FindConflictingGuidNames(gs))
# Log errors for anything within the package under test
for er in Errors[:]:
InMyPackage = False
for a in er.entries:
if abs_pkg_path in a.absfilepath:
InMyPackage = True
break
if(not InMyPackage):
Errors.remove(er)
else:
logging.error(str(er))
tc.LogStdError(str(er))
# add result to test case
overall_status = len(Errors)
if overall_status is not 0:
tc.SetFailed("GuidCheck {0} Failed. Errors {1}".format(
packagename, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status
class ErrorEntry():
""" Custom/private class for reporting errors in the GuidList
"""
def __init__(self, errortype):
self.type = errortype # 'guid' or 'name' depending on error type
self.entries = [] # GuidListEntry that are in error condition
def __str__(self):
a = f"Error Duplicate {self.type}: "
if(self.type == "guid"):
a += f" {self.entries[0].guid}"
elif(self.type == "name"):
a += f" {self.entries[0].name}"
a += f" ({len(self.entries)})\n"
for e in self.entries:
a += "\t" + str(e) + "\n"
return a

View File

@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to check guid uniqueness
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Guid Check Test",
"module": "GuidCheck"
}

View File

@ -0,0 +1,80 @@
# Guid Check Plugin
This CiBuildPlugin scans all the files in a code tree to find all the GUID
definitions. After collection it will then look for duplication in the package
under test. Uniqueness of all GUIDs are critical within the UEFI environment.
Duplication can cause numerous issues including locating the wrong data
structure, calling the wrong function, or decoding the wrong data members.
Currently Scanned:
* INF files are scanned for there Module guid
* DEC files are scanned for all of their Protocols, PPIs, and Guids as well as
the one package GUID.
Any GUID value being equal to two names or even just defined in two files is
considered an error unless in the ignore list.
Any GUID name that is found more than once is an error unless all occurrences
are Module GUIDs. Since the Module GUID is assigned to the Module name it is
common to have numerous versions of the same module named the same.
## Configuration
The plugin has numerous configuration options to support the UEFI codebase.
``` yaml
"GuidCheck": {
"IgnoreGuidName": [],
"IgnoreGuidValue": [],
"IgnoreFoldersAndFiles": [],
"IgnoreDuplicates": []
}
```
### IgnoreGuidName
This list allows strings in two formats.
* _GuidName_
* This will remove any entry with this GuidName from the list of GUIDs
therefore ignoring any error associated with this name.
* _GuidName=GuidValue_
* This will also ignore the GUID by name but only if the value equals the
GuidValue.
* GuidValue should be in registry format.
* This is the suggested format to use as it will limit the ignore to only the
defined case.
### IgnoreGuidValue
This list allows strings in guid registry format _GuidValue_.
* This will remove any entry with this GuidValue from the list of GUIDs
therefore ignoring any error associated with this value.
* GuidValue must be in registry format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
### IgnoreFoldersAndFiles
This supports .gitignore file and folder matching strings including wildcards
* Any folder or file ignored will not be parsed and therefore any GUID defined
will be ignored.
* The plugin will always ignores the following ["/Build", "/Conf"]
### IgnoreDuplicates
This supports strings in the format of _GuidName_=_GuidName_=_GuidName_
* For the error with the GuidNames to be ignored the list must match completely
with what is found during the code scan.
* For example if there are two GUIDs that are by design equal within the code
tree then it should be _GuidName_=_GuidName_
* If instead there are three GUIDs then it must be
_GuidName_=_GuidName_=_GuidName_
* This is the best ignore list to use because it is the most strict and will
catch new problems when new conflicts are introduced.
* There are numerous places in the UEFI specification in which two GUID names
are assigned the same value. These names should be set in this ignore list so
that they don't cause an error but any additional duplication would still be
caught.

View File

@ -0,0 +1,153 @@
# @file LibraryClassCheck.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import os
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.uefi.edk2.parsers.dec_parser import DecParser
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toolext.environment.var_dict import VarDict
class LibraryClassCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that scans the code tree and library classes for undeclared
files
Configuration options:
"LibraryClassCheck": {
IgnoreHeaderFile: [], # Ignore a file found on disk
IgnoreLibraryClass: [] # Ignore a declaration found in dec file
}
"""
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
"""
return ("Check library class declarations in " + packagename, packagename + ".LibraryClassCheck")
def __GetPkgDec(self, rootpath):
try:
allEntries = os.listdir(rootpath)
for entry in allEntries:
if entry.lower().endswith(".dec"):
return(os.path.join(rootpath, entry))
except Exception:
logging.error("Unable to find DEC for package:{0}".format(rootpath))
return None
##
# External function of plugin. This function is used to perform the task of the MuBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
overall_status = 0
LibraryClassIgnore = []
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
abs_dec_path = self.__GetPkgDec(abs_pkg_path)
wsr_dec_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dec_path)
if abs_dec_path is None or wsr_dec_path is "" or not os.path.isfile(abs_dec_path):
tc.SetSkipped()
tc.LogStdError("No DEC file {0} in package {1}".format(abs_dec_path, abs_pkg_path))
return -1
# Get all include folders
dec = DecParser()
dec.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList)
dec.ParseFile(wsr_dec_path)
AllHeaderFiles = []
for includepath in dec.IncludePaths:
## Get all header files in the library folder
AbsLibraryIncludePath = os.path.join(abs_pkg_path, includepath, "Library")
if(not os.path.isdir(AbsLibraryIncludePath)):
continue
hfiles = self.WalkDirectoryForExtension([".h"], AbsLibraryIncludePath)
hfiles = [os.path.relpath(x,abs_pkg_path) for x in hfiles] # make package root relative path
hfiles = [x.replace("\\", "/") for x in hfiles] # make package relative path
AllHeaderFiles.extend(hfiles)
if len(AllHeaderFiles) == 0:
tc.SetSkipped()
tc.LogStdError(f"No Library include folder in any Include path")
return -1
# Remove ignored paths
if "IgnoreHeaderFile" in pkgconfig:
for a in pkgconfig["IgnoreHeaderFile"]:
try:
tc.LogStdOut("Ignoring Library Header File {0}".format(a))
AllHeaderFiles.remove(a)
except:
tc.LogStdError("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a))
logging.info("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a))
if "IgnoreLibraryClass" in pkgconfig:
LibraryClassIgnore = pkgconfig["IgnoreLibraryClass"]
## Attempt to find library classes
for lcd in dec.LibraryClasses:
## Check for correct file path separator
if "\\" in lcd.path:
tc.LogStdError("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path))
logging.error("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path))
overall_status += 1
continue
if lcd.name in LibraryClassIgnore:
tc.LogStdOut("Ignoring Library Class Name {0}".format(lcd.name))
LibraryClassIgnore.remove(lcd.name)
continue
logging.debug(f"Looking for Library Class {lcd.path}")
try:
AllHeaderFiles.remove(lcd.path)
except ValueError:
tc.LogStdError(f"Library {lcd.name} with path {lcd.path} not found in package filesystem")
logging.error(f"Library {lcd.name} with path {lcd.path} not found in package filesystem")
overall_status += 1
## any remaining AllHeaderFiles are not described in DEC
for h in AllHeaderFiles:
tc.LogStdError(f"Library Header File {h} not declared in package DEC {wsr_dec_path}")
logging.error(f"Library Header File {h} not declared in package DEC {wsr_dec_path}")
overall_status += 1
## Warn about any invalid library class names in the ignore list
for r in LibraryClassIgnore:
tc.LogStdError("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r))
logging.info("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r))
# If XML object exists, add result
if overall_status is not 0:
tc.SetFailed("LibraryClassCheck {0} Failed. Errors {1}".format(wsr_dec_path, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status

View File

@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to check that all library classes are declared correctly in dec file
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Library Class Check Test",
"module": "LibraryClassCheck"
}

View File

@ -0,0 +1,25 @@
# Library Class Check Plugin
This CiBuildPlugin scans at all library header files found in the `Library`
folders in all of the package's declared include directories and ensures that
all files have a matching LibraryClass declaration in the DEC file for the
package. Any missing declarations will cause a failure.
## Configuration
The plugin has a few configuration options to support the UEFI codebase.
``` yaml
"LibraryClassCheck": {
IgnoreHeaderFile: [], # Ignore a file found on disk
IgnoreLibraryClass: [] # Ignore a declaration found in dec file
}
```
### IgnoreHeaderFile
Ignore a file found on disk
### IgnoreLibraryClass
Ignore a declaration found in dec file

View File

@ -0,0 +1,127 @@
# Spell Check Plugin
This CiBuildPlugin scans all the files in a given package and checks for
spelling errors.
This plugin requires NodeJs and cspell. If the plugin doesn't find its required
tools then it will mark the test as skipped.
* NodeJS: https://nodejs.org/en/
* cspell: https://www.npmjs.com/package/cspell
* Src and doc available: https://github.com/streetsidesoftware/cspell
## Configuration
The plugin has a few configuration options to support the UEFI codebase.
``` yaml
"SpellCheck": {
"AuditOnly": False, # If True, log all errors and then mark as skipped
"IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files
"ExtendWords": [], # words to extend to the dictionary for this package
"IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore
"AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)
}
```
### AuditOnly
Boolean - Default is False.
If True run the test in an Audit only mode which will log all errors but instead
of failing the build it will set the test as skipped. This allows visibility
into the failures without breaking the build.
### IgnoreFiles
This supports .gitignore file and folder matching strings including wildcards
* All files will be parsed regardless but then any spelling errors found within
ignored files will not be reported as an error.
* Errors in ignored files will still be output to the test results as
informational comments.
### ExtendWords
This list allows words to be added to the dictionary for the spell checker when
this package is tested. These follow the rules of the cspell config words field.
### IgnoreStandardPaths
This plugin by default will check the below standard paths. If the package
would like to ignore any of them list that here.
```python
[
# C source
"*.c",
"*.h",
# Assembly files
"*.nasm",
"*.asm",
"*.masm",
"*.s",
# ACPI source language
"*.asl",
# Edk2 build files
"*.dsc", "*.dec", "*.fdf", "*.inf",
# Documentation files
"*.md", "*.txt"
]
```
### AdditionalIncludePaths
If the package would to add additional path patterns to be included in
spellchecking they can be defined here.
## Other configuration
In the cspell.base.json there are numerous other settings configured. There is
no support to override these on a per package basis but future features could
make this available. One interesting configuration option is `minWordLength`.
Currently it is set to _5_ which means all 2,3, and 4 letter words will be
ignored. This helps minimize the number of technical acronyms, register names,
and other UEFI specific values that must be ignored.
## False positives
The cspell dictionary is not perfect and there are cases where technical words
or acronyms are not found in the dictionary. There are three ways to resolve
false positives and the choice for which method should be based on how broadly
the word should be accepted.
### CSpell Base Config file
If the change should apply to all UEFI code and documentation then it should be
added to the base config file `words` section. The base config file is adjacent
to this file and titled `cspell.base.json`. This is a list of accepted words
for all spell checking operations on all packages.
### Package Config
In the package `*.ci.yaml` file there is a `SpellCheck` config section. This
section allows files to be ignored as well as words that should be considered
valid for all files within this package. Add the desired words to the
"ExtendedWords" member.
### In-line File
CSpell supports numerous methods to annotate your files to ignore words,
sections, etc. This can be found in CSpell documentation. Suggestion here is
to use a c-style comment at the top of the file to add words that should be
ignored just for this file. Obviously this has the highest maintenance cost so
it should only be used for file unique words.
``` c
// spell-checker:ignore unenroll, word2, word3
```
or
```ini
# spell-checker:ignore unenroll, word2, word3
```

View File

@ -0,0 +1,216 @@
# @file SpellCheck.py
#
# An edk2-pytool based plugin wrapper for cspell
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import json
import yaml
from io import StringIO
import os
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.utility_functions import RunCmd
from edk2toolext.environment.var_dict import VarDict
from edk2toollib.gitignore_parser import parse_gitignore_lines
from edk2toolext.environment import version_aggregator
class SpellCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that uses the cspell node module to scan the files
from the package being tested for spelling errors. The plugin contains
the base cspell.json file then thru the configuration options other settings
can be changed or extended.
Configuration options:
"SpellCheck": {
"AuditOnly": False, # Don't fail the build if there are errors. Just log them
"IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files
"ExtendWords": [], # words to extend to the dictionary for this package
"IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore
"AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)
}
"""
#
# A package can remove any of these using IgnoreStandardPaths
#
STANDARD_PLUGIN_DEFINED_PATHS = ["*.c", "*.h",
"*.nasm", "*.asm", "*.masm", "*.s",
"*.asl",
"*.dsc", "*.dec", "*.fdf", "*.inf",
"*.md", "*.txt"
]
def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
"""
return ("Spell check files in " + packagename, packagename + ".SpellCheck")
##
# External function of plugin. This function is used to perform the task of the CiBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
Errors = []
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
packagename)
if abs_pkg_path is None:
tc.SetSkipped()
tc.LogStdError("No package {0}".format(packagename))
return -1
# check for node
return_buffer = StringIO()
ret = RunCmd("node", "--version", outstream=return_buffer)
if (ret != 0):
tc.SetSkipped()
tc.LogStdError("NodeJs not installed. Test can't run")
logging.warning("NodeJs not installed. Test can't run")
return -1
node_version = return_buffer.getvalue().strip() # format vXX.XX.XX
tc.LogStdOut(f"Node version: {node_version}")
version_aggregator.GetVersionAggregator().ReportVersion(
"NodeJs", node_version, version_aggregator.VersionTypes.INFO)
# Check for cspell
return_buffer = StringIO()
ret = RunCmd("cspell", "--version", outstream=return_buffer)
if (ret != 0):
tc.SetSkipped()
tc.LogStdError("cspell not installed. Test can't run")
logging.warning("cspell not installed. Test can't run")
return -1
cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX
tc.LogStdOut(f"CSpell version: {cspell_version}")
version_aggregator.GetVersionAggregator().ReportVersion(
"CSpell", cspell_version, version_aggregator.VersionTypes.INFO)
package_relative_paths_to_spell_check = SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS
#
# Allow the ci.yaml to remove any of the above standard paths
#
if("IgnoreStandardPaths" in pkgconfig):
for a in pkgconfig["IgnoreStandardPaths"]:
if(a in package_relative_paths_to_spell_check):
tc.LogStdOut(
f"ignoring standard path due to ci.yaml ignore: {a}")
package_relative_paths_to_spell_check.remove(a)
else:
tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}")
#
# check for any additional include paths defined by package config
#
if("AdditionalIncludePaths" in pkgconfig):
package_relative_paths_to_spell_check.extend(
pkgconfig["AdditionalIncludePaths"])
#
# Make the path string for cspell to check
#
relpath = os.path.relpath(abs_pkg_path)
cpsell_paths = " ".join(
[f"{relpath}/**/{x}" for x in package_relative_paths_to_spell_check])
# Make the config file
config_file_path = os.path.join(
Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json")
mydir = os.path.dirname(os.path.abspath(__file__))
# load as yaml so it can have comments
base = os.path.join(mydir, "cspell.base.yaml")
with open(base, "r") as i:
config = yaml.safe_load(i)
if("ExtendWords" in pkgconfig):
config["words"].extend(pkgconfig["ExtendWords"])
with open(config_file_path, "w") as o:
json.dump(config, o) # output as json so compat with cspell
All_Ignores = []
# Parse the config for other ignores
if "IgnoreFiles" in pkgconfig:
All_Ignores.extend(pkgconfig["IgnoreFiles"])
# spell check all the files
ignore = parse_gitignore_lines(All_Ignores, os.path.join(
abs_pkg_path, "nofile.txt"), abs_pkg_path)
# result is a list of strings like this
# C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule)
EasyFix = []
results = self._check_spelling(cpsell_paths, config_file_path)
for r in results:
path, _, word = r.partition(" - Unknown word ")
if len(word) == 0:
# didn't find pattern
continue
pathinfo = path.rsplit(":", 2) # remove the line no info
if(ignore(pathinfo[0])): # check against ignore list
tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}")
continue
# real error
EasyFix.append(word.strip().strip("()"))
Errors.append(r)
# Log all errors tc StdError
for l in Errors:
tc.LogStdError(l.strip())
# Helper - Log the syntax needed to add these words to dictionary
if len(EasyFix) > 0:
EasyFix = sorted(set(a.lower() for a in EasyFix))
tc.LogStdOut("\n Easy fix:")
OneString = "If these are not errors add this to your ci.yaml file.\n"
OneString += '"SpellCheck": {\n "ExtendWords": ['
for a in EasyFix:
tc.LogStdOut(f'\n"{a}",')
OneString += f'\n "{a}",'
logging.info(OneString.rstrip(",") + '\n ]\n}')
# add result to test case
overall_status = len(Errors)
if overall_status != 0:
if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]:
# set as skipped if AuditOnly
tc.SetSkipped()
return -1
else:
tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format(
packagename, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status
def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []:
output = StringIO()
ret = RunCmd(
"cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output)
if ret == 0:
return []
else:
return output.getvalue().strip().splitlines()

View File

@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to check spelling
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Spell Check Test",
"module": "SpellCheck"
}

View File

@ -0,0 +1,165 @@
## @file
# CSpell configuration
#
# Copyright (c) Microsoft Corporation
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"version": "0.1",
"language": "en",
"dictionaries": [
"companies ",
"softwareTerms",
"python",
"cpp"
],
"ignorePaths": [
"*.pdb",
"**/*_extdep/**",
"*.pdf",
"*.exe",
"*.jpg"
],
"minWordLength": 5,
"allowCompoundWords": false,
"ignoreWords": [
"muchange"
],
"words": [
"MTRRs",
"Microarchitecture",
"Goldmont",
"cpuid",
"mwait",
"cstate",
"smram",
"scrtm",
"smbus",
"selftest",
"socket",
"MMRAM",
"qword",
"ENDBR",
"SMBASE",
"FXSAVE",
"FXRSTOR",
"RDRAND",
"IOAPIC",
"ATAPI",
"movsb",
"iretw",
"XENSTORE",
"cdrom",
"oprom",
"oproms",
"varstore",
"EKU",
"ascii",
"nmake",
"NVDIMM",
"nasmb",
"Mtftp",
"Hypercall",
"hypercalls",
"IOMMU",
"QEMU",
"qemus",
"OVMF",
"tiano",
"tianocore",
"edkii",
"coreboot",
"uefipayload",
"bootloader",
"bootloaders",
"mdepkg",
"skuid",
"dxefv",
"toolchain",
"libraryclass",
"preboot",
"pythonpath",
"cygpath",
"nuget",
"basetools",
"prepi",
"OPTEE",
"stringid",
"peims",
"memmap",
"guids",
"uuids",
"smbios",
"certdb",
"certdbv",
"EfiSigList",
"depex",
"IHANDLE",
"Virtio",
"Mbytes",
"Citrix",
"initrd",
"semihost",
"Semihosting",
"Trustzone",
"Fastboot",
"framebuffer",
"genfw",
"TTYTERM",
"miniport",
"LFENCE",
"PCANSI",
"submodule",
"submodules",
"brotli",
"PCCTS",
"softfloat",
"whitepaper",
"ACPICA",
"plugfest",
"bringup",
"formset", #VFR
"ideqvallist",
"numberof",
"oneof",
"endformset",
"endnumeric",
"endoneof",
"disableif",
"guidid",
"classguid",
"efivarstore",
"formsetguid",
"formid",
"suppressif",
"grayoutif",
"ideqval",
"endform",
"endcheckbox",
"questionid",
"questionref",
"enddate",
"endstring",
"guidop",
"endguidop",
"langdef",
"dynamicex",
"tokenspace",
"tokenguid",
"pcd's", #seems like cspell bug
"peim's",
"autogen",
"Disasm",
"Torito",
"SRIOV",
"MRIOV",
"UARTs",
"Consplitter", # common module in UEFI
"FIFOs",
"ACPINVS",
"Endof", # due to of not being uppercase
"bootability",
"Sdhci",
"inmodule",
]
}