tyler.durden
/
test262
mirror of https://github.com/tc39/test262.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug 1472: Adding JUnit Compatible XML and Logfile support for Python test runner

This commit is contained in:
Prashanth Srinivasan 2013-06-10 11:51:24 -04:00
parent 62f3c563fe
commit 87605ba3b5
2 changed files with 109 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/config/excludelist.xml
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8" ?>
<excludeList>
<test id="example">Some description </test>
<test id="example"><reason>Some description </reason></test>
</excludeList>

114
tools/packaging/test262.py
View File

@ -23,6 +23,8 @@ import datetime
import shutil
import json
import stat
import xml.etree.ElementTree as xmlj
import unicodedata
from parseTestRecord import parseTestRecord, stripHeader
@ -43,6 +45,7 @@ if not os.path.exists(EXCLUDED_FILENAME):
" %s, showing which tests have been disabled!" % EXCLUDED_FILENAME
sys.exit(1)
EXCLUDE_LIST = xml.dom.minidom.parse(EXCLUDED_FILENAME)
EXCLUDE_REASON = EXCLUDE_LIST.getElementsByTagName("reason")
EXCLUDE_LIST = EXCLUDE_LIST.getElementsByTagName("test")
EXCLUDE_LIST = [x.getAttribute("id") for x in EXCLUDE_LIST]
@ -66,6 +69,8 @@ def BuildOptions():
# to "both"
result.add_option("--unmarked_default", default="non_strict",
help="default mode for tests of unspecified strictness")
result.add_option("--logname", help="Filename to save stdout to")
result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
result.add_option("--loglevel", default="warning",
help="sets log level to debug, info, warning, error, or critical")
return result
@ -159,6 +164,35 @@ class TestResult(object):
else:
print "%s passed in %s" % (name, mode)
def XmlAssemble(self, result):
test_name = self.case.GetName()
test_mode = self.case.GetMode()
testCaseElement = xmlj.Element("testcase")
testpath = self.TestPathManipulation(test_name)
testCaseElement.attrib["classname"] = "%s.%s" % (testpath[0] , testpath[1])
testCaseElement.attrib["name"] = "%s %s" % (testpath[2].replace('.','_') , test_mode)
if self.HasUnexpectedOutcome():
failureElement = xmlj.Element("failure")
out = self.stdout.strip().decode('utf-8')
err = self.stderr.strip().decode('utf-8')
if len(out) > 0:
failureElement.text = out
if len(err) > 0:
failureElement.text = err
testCaseElement.append(failureElement)
return testCaseElement
def TestPathManipulation(self, test_name):
testdirlist = test_name.split('/')
testcase = testdirlist.pop()
testclass = testdirlist.pop()
testclass = testclass.replace('.','_')
if len(testdirlist) >= 1:
testpackage = testdirlist.pop(0)
else:
testpackage = testclass
return(testpackage,testclass,testcase)
def HasFailed(self):
return self.exit_code != 0
@ -296,7 +330,6 @@ def MakePlural(n):
else:
return (n, "s")
class TestSuite(object):
def __init__(self, root, strict_only, non_strict_only, unmarked_default):
@ -377,63 +410,125 @@ class TestSuite(object):
logging.info("Done listing tests")
return cases
def PrintSummary(self, progress):
def PrintSummary(self, progress, logfile):
print
if logfile:
self.logf.write("=== Summary === \n")
print "=== Summary ==="
count = progress.count
succeeded = progress.succeeded
failed = progress.failed
if logfile:
self.logf.write(" - Ran %i test%s \n" % MakePlural(count))
print " - Ran %i test%s" % MakePlural(count)
if progress.failed == 0:
if logfile:
self.logf.write(" - All tests succeeded \n")
print " - All tests succeeded"
else:
percent = ((100.0 * succeeded) / count,)
if logfile:
self.logf.write(" - Passed %i test%s (%.1f%%)\n" % (MakePlural(succeeded) + percent))
print " - Passed %i test%s (%.1f%%)" % (MakePlural(succeeded) + percent)
percent = ((100.0 * failed) / count,)
if logfile:
self.logf.write(" - Failed %i test%s (%.1f%%) \n" % (MakePlural(failed) + percent))
print " - Failed %i test%s (%.1f%%)" % (MakePlural(failed) + percent)
positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
negative = [c for c in progress.failed_tests if c.case.IsNegative()]
if len(positive) > 0:
print
if logfile:
self.logf.write("Failed Tests \n")
print "Failed tests"
for result in positive:
if logfile:
self.logf.write(" %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
print " %s in %s" % (result.case.GetName(), result.case.GetMode())
if len(negative) > 0:
print
print "Expected to fail but passed ---"
for result in negative:
if logfile:
self.logfile.append(" %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
print " %s in %s" % (result.case.GetName(), result.case.GetMode())
def PrintFailureOutput(self, progress):
def PrintFailureOutput(self, progress, logfile):
for result in progress.failed_tests:
if logfile:
self.WriteLog(result)
print
result.ReportOutcome(False)
def Run(self, command_template, tests, print_summary, full_summary):
def Run(self, command_template, tests, print_summary, full_summary, logname, junitfile):
if not "{{path}}" in command_template:
command_template += " {{path}}"
cases = self.EnumerateTests(tests)
if len(cases) == 0:
ReportError("No tests to run")
progress = ProgressIndicator(len(cases))
if logname:
self.logf = open(logname, "w")
if junitfile:
self.outfile = open(junitfile, "w")
TestSuiteElement = xmlj.Element("testsuite")
TestSuiteElement.attrib["name "] = "test262"
for x in range(len(EXCLUDE_LIST)):
if self.ShouldRun (unicode(EXCLUDE_LIST[x].encode('utf-8','ignore')), tests):
SkipCaseElement = xmlj.Element("testcase")
SkipCaseElement.attrib["classname"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
SkipCaseElement.attrib["name"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
SkipElement = xmlj.Element("skipped")
SkipElement.attrib["message"] = unicode(EXCLUDE_REASON[x].firstChild.nodeValue)
SkipCaseElement.append(SkipElement)
TestSuiteElement.append(SkipCaseElement)
for case in cases:
result = case.Run(command_template)
if junitfile:
TestCaseElement = result.XmlAssemble(result)
TestSuiteElement.append(TestCaseElement)
if case == cases[len(cases)-1]:
xmlj.ElementTree(TestSuiteElement).write(junitfile, "UTF-8")
if logname:
self.WriteLog(result)
progress.HasRun(result)
if print_summary:
self.PrintSummary(progress)
self.PrintSummary(progress, logname)
if full_summary:
self.PrintFailureOutput(progress)
self.PrintFailureOutput(progress, logname)
else:
print
print "Use --full-summary to see output from failed tests"
print
def WriteLog(self, result):
name = result.case.GetName()
mode = result.case.GetMode()
if result.HasUnexpectedOutcome():
if result.case.IsNegative():
self.logf.write("=== %s was expected to fail in %s, but didn't === \n" % (name, mode))
else:
self.logf.write("=== %s failed in %s === \n" % (name, mode))
out = result.stdout.strip()
if len(out) > 0:
self.logf.write("--- output --- \n %s" % out)
err = result.stderr.strip()
if len(err) > 0:
self.logf.write("--- errors --- \n %s" % err)
self.logf.write("=== \n")
elif result.case.IsNegative():
self.logf.write("%s failed in %s as expected \n" % (name, mode))
else:
self.logf.write("%s passed in %s \n" % (name, mode))
def Print(self, tests):
cases = self.EnumerateTests(tests)
if len(cases) > 0:
cases[0].Print()
def Main():
parser = BuildOptions()
(options, args) = parser.parse_args()
@ -458,8 +553,9 @@ def Main():
else:
test_suite.Run(options.command, args,
options.summary or options.full_summary,
options.full_summary)
options.full_summary,
options.logname,
options.junitname)
if __name__ == '__main__':
try: