mirror of https://github.com/tc39/test262.git
647 lines
20 KiB
Python
Executable File
647 lines
20 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2009 the Sputnik authors. All rights reserved.
|
|
# This code is governed by the BSD license found in the LICENSE file.
|
|
|
|
# This is derived from sputnik.py, the Sputnik console test runner,
|
|
# with elements from packager.py, which is separately
|
|
# copyrighted. TODO: Refactor so there is less duplication between
|
|
# test262.py and packager.py.
|
|
|
|
|
|
import logging
|
|
import optparse
|
|
import os
|
|
from os import path
|
|
import platform
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import xml.dom.minidom
|
|
import datetime
|
|
import shutil
|
|
import json
|
|
import stat
|
|
import xml.etree.ElementTree as xmlj
|
|
import unicodedata
|
|
from collections import Counter
|
|
|
|
|
|
from parseTestRecord import parseTestRecord, stripHeader
|
|
|
|
from packagerConfig import *
|
|
|
|
class Test262Error(Exception):
|
|
def __init__(self, message):
|
|
self.message = message
|
|
|
|
def ReportError(s):
|
|
raise Test262Error(s)
|
|
|
|
|
|
|
|
if not os.path.exists(EXCLUDED_FILENAME):
|
|
print "Cannot generate (JSON) test262 tests without a file," + \
|
|
" %s, showing which tests have been disabled!" % EXCLUDED_FILENAME
|
|
sys.exit(1)
|
|
EXCLUDE_LIST = xml.dom.minidom.parse(EXCLUDED_FILENAME)
|
|
EXCLUDE_REASON = EXCLUDE_LIST.getElementsByTagName("reason")
|
|
EXCLUDE_LIST = EXCLUDE_LIST.getElementsByTagName("test")
|
|
EXCLUDE_LIST = [x.getAttribute("id") for x in EXCLUDE_LIST]
|
|
|
|
|
|
def BuildOptions():
|
|
result = optparse.OptionParser()
|
|
result.add_option("--command", default=None, help="The command-line to run")
|
|
result.add_option("--tests", default=path.abspath('.'),
|
|
help="Path to the tests")
|
|
result.add_option("--cat", default=False, action="store_true",
|
|
help="Print packaged test code that would be run")
|
|
result.add_option("--summary", default=False, action="store_true",
|
|
help="Print summary after running tests")
|
|
result.add_option("--full-summary", default=False, action="store_true",
|
|
help="Print summary and test output after running tests")
|
|
result.add_option("--strict_only", default=False, action="store_true",
|
|
help="Test only strict mode")
|
|
result.add_option("--non_strict_only", default=False, action="store_true",
|
|
help="Test only non-strict mode")
|
|
result.add_option("--unmarked_default", default="both",
|
|
help="default mode for tests of unspecified strictness")
|
|
result.add_option("--logname", help="Filename to save stdout to")
|
|
result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
|
|
result.add_option("--loglevel", default="warning",
|
|
help="sets log level to debug, info, warning, error, or critical")
|
|
result.add_option("--print-handle", default="print", help="Command to print from console")
|
|
result.add_option("--list-includes", default=False, action="store_true",
|
|
help="List includes required by tests")
|
|
return result
|
|
|
|
|
|
def ValidateOptions(options):
|
|
if not options.command:
|
|
ReportError("A --command must be specified.")
|
|
if not path.exists(options.tests):
|
|
ReportError("Couldn't find test path '%s'" % options.tests)
|
|
|
|
|
|
placeHolderPattern = re.compile(r"\{\{(\w+)\}\}")
|
|
|
|
|
|
def IsWindows():
|
|
p = platform.system()
|
|
return (p == 'Windows') or (p == 'Microsoft')
|
|
|
|
|
|
class TempFile(object):
|
|
|
|
def __init__(self, suffix="", prefix="tmp", text=False):
|
|
self.suffix = suffix
|
|
self.prefix = prefix
|
|
self.text = text
|
|
self.fd = None
|
|
self.name = None
|
|
self.is_closed = False
|
|
self.Open()
|
|
|
|
def Open(self):
|
|
(self.fd, self.name) = tempfile.mkstemp(
|
|
suffix = self.suffix,
|
|
prefix = self.prefix,
|
|
text = self.text)
|
|
|
|
def Write(self, str):
|
|
os.write(self.fd, str)
|
|
|
|
def Read(self):
|
|
f = file(self.name)
|
|
result = f.read()
|
|
f.close()
|
|
return result
|
|
|
|
def Close(self):
|
|
if not self.is_closed:
|
|
self.is_closed = True
|
|
os.close(self.fd)
|
|
|
|
def Dispose(self):
|
|
try:
|
|
self.Close()
|
|
os.unlink(self.name)
|
|
except OSError, e:
|
|
logging.error("Error disposing temp file: %s", str(e))
|
|
|
|
|
|
class TestResult(object):
|
|
|
|
def __init__(self, exit_code, stdout, stderr, case):
|
|
self.exit_code = exit_code
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
self.case = case
|
|
|
|
def ReportOutcome(self, long_format):
|
|
name = self.case.GetName()
|
|
mode = self.case.GetMode()
|
|
if self.HasUnexpectedOutcome():
|
|
if self.case.IsNegative():
|
|
print "=== %s was expected to fail in %s, but didn't ===" % (name, mode)
|
|
print "--- expected error: %s ---\n" % self.case.GetNegative()
|
|
else:
|
|
if long_format:
|
|
print "=== %s failed in %s ===" % (name, mode)
|
|
else:
|
|
print "%s in %s: " % (name, mode)
|
|
self.WriteOutput(sys.stdout)
|
|
if long_format:
|
|
print "==="
|
|
elif self.case.IsNegative():
|
|
print "%s failed in %s as expected" % (name, mode)
|
|
else:
|
|
print "%s passed in %s" % (name, mode)
|
|
|
|
def WriteOutput(self, target):
|
|
out = self.stdout.strip()
|
|
if len(out) > 0:
|
|
target.write("--- output --- \n %s" % out)
|
|
err = self.stderr.strip()
|
|
if len(err) > 0:
|
|
target.write("--- errors --- \n %s" % err)
|
|
|
|
# This is a way to make the output from the "whitespace" tests into valid XML
|
|
def SafeFormat(self, msg):
|
|
try:
|
|
msg = msg.encode(encoding='ascii', errors='strict')
|
|
msg = msg.replace('\u000Bx', '?')
|
|
msg = msg.replace('\u000Cx', '?')
|
|
except:
|
|
return 'Output contained invalid characters'
|
|
|
|
def XmlAssemble(self, result):
|
|
test_name = self.case.GetName()
|
|
test_mode = self.case.GetMode()
|
|
testCaseElement = xmlj.Element("testcase")
|
|
testpath = self.TestPathManipulation(test_name)
|
|
testCaseElement.attrib["classname"] = "%s.%s" % (testpath[0] , testpath[1])
|
|
testCaseElement.attrib["name"] = "%s %s" % (testpath[2].replace('.','_') , test_mode)
|
|
if self.HasUnexpectedOutcome():
|
|
failureElement = xmlj.Element("failure")
|
|
out = self.stdout.strip().decode('utf-8')
|
|
err = self.stderr.strip().decode('utf-8')
|
|
if len(out) > 0:
|
|
failureElement.text = self.SafeFormat(out)
|
|
if len(err) > 0:
|
|
failureElement.text = self.SafeFormat(err)
|
|
testCaseElement.append(failureElement)
|
|
return testCaseElement
|
|
|
|
def TestPathManipulation(self, test_name):
|
|
testdirlist = test_name.split('/')
|
|
testcase = testdirlist.pop()
|
|
testclass = testdirlist.pop()
|
|
testclass = testclass.replace('.','_')
|
|
if len(testdirlist) >= 1:
|
|
testpackage = testdirlist.pop(0)
|
|
else:
|
|
testpackage = testclass
|
|
return(testpackage,testclass,testcase)
|
|
|
|
def HasFailed(self):
|
|
return self.exit_code != 0
|
|
|
|
def AsyncHasFailed(self):
|
|
return 'Test262:AsyncTestComplete' not in self.stdout
|
|
|
|
def HasUnexpectedOutcome(self):
|
|
if self.case.IsAsyncTest():
|
|
return self.AsyncHasFailed() or self.HasFailed()
|
|
elif self.case.IsNegative():
|
|
return not (self.HasFailed() and self.case.NegativeMatch(self.GetErrorOutput()))
|
|
else:
|
|
return self.HasFailed()
|
|
|
|
def GetErrorOutput(self):
|
|
if len(self.stderr) != 0:
|
|
return self.stderr
|
|
return self.stdout
|
|
|
|
|
|
class TestCase(object):
|
|
|
|
def __init__(self, suite, name, full_path, strict_mode):
|
|
self.suite = suite
|
|
self.name = name
|
|
self.full_path = full_path
|
|
self.strict_mode = strict_mode
|
|
f = open(self.full_path)
|
|
self.contents = f.read()
|
|
f.close()
|
|
testRecord = parseTestRecord(self.contents, name)
|
|
self.test = testRecord["test"]
|
|
del testRecord["test"]
|
|
del testRecord["header"]
|
|
testRecord.pop("commentary", None) # do not throw if missing
|
|
self.testRecord = testRecord;
|
|
|
|
self.validate()
|
|
|
|
def NegativeMatch(self, stderr):
|
|
neg = re.compile(self.GetNegative())
|
|
return re.search(neg, stderr)
|
|
|
|
def GetNegative(self):
|
|
return self.testRecord['negative']
|
|
|
|
def GetName(self):
|
|
return path.join(*self.name)
|
|
|
|
def GetMode(self):
|
|
if self.strict_mode:
|
|
return "strict mode"
|
|
else:
|
|
return "non-strict mode"
|
|
|
|
def GetPath(self):
|
|
return self.name
|
|
|
|
def IsNegative(self):
|
|
return 'negative' in self.testRecord
|
|
|
|
def IsOnlyStrict(self):
|
|
return 'onlyStrict' in self.testRecord
|
|
|
|
def IsNoStrict(self):
|
|
return 'noStrict' in self.testRecord or self.IsRaw()
|
|
|
|
def IsRaw(self):
|
|
return 'raw' in self.testRecord
|
|
|
|
def IsAsyncTest(self):
|
|
return 'async' in self.testRecord
|
|
|
|
def GetIncludeList(self):
|
|
if self.testRecord.get('includes'):
|
|
return self.testRecord['includes']
|
|
return []
|
|
|
|
def GetAdditionalIncludes(self):
|
|
return '\n'.join([self.suite.GetInclude(include) for include in self.GetIncludeList()])
|
|
|
|
def GetSource(self):
|
|
if self.IsRaw():
|
|
return self.test
|
|
|
|
source = self.suite.GetInclude("sta.js") + \
|
|
self.suite.GetInclude("cth.js") + \
|
|
self.suite.GetInclude("assert.js")
|
|
|
|
if self.IsAsyncTest():
|
|
source = source + \
|
|
self.suite.GetInclude("timer.js") + \
|
|
self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle)
|
|
|
|
source = source + \
|
|
self.GetAdditionalIncludes() + \
|
|
self.test + '\n'
|
|
|
|
if self.strict_mode:
|
|
source = '"use strict";\nvar strict_mode = true;\n' + source
|
|
else:
|
|
# add comment line so line numbers match in both strict and non-strict version
|
|
source = '//"no strict";\nvar strict_mode = false;\n' + source
|
|
|
|
return source
|
|
|
|
def InstantiateTemplate(self, template, params):
|
|
def GetParameter(match):
|
|
key = match.group(1)
|
|
return params.get(key, match.group(0))
|
|
return placeHolderPattern.sub(GetParameter, template)
|
|
|
|
def Execute(self, command):
|
|
if IsWindows():
|
|
args = '%s' % command
|
|
else:
|
|
args = command.split(" ")
|
|
stdout = TempFile(prefix="test262-out-")
|
|
stderr = TempFile(prefix="test262-err-")
|
|
try:
|
|
logging.info("exec: %s", str(args))
|
|
process = subprocess.Popen(
|
|
args,
|
|
shell = IsWindows(),
|
|
stdout = stdout.fd,
|
|
stderr = stderr.fd
|
|
)
|
|
code = process.wait()
|
|
out = stdout.Read()
|
|
err = stderr.Read()
|
|
finally:
|
|
stdout.Dispose()
|
|
stderr.Dispose()
|
|
return (code, out, err)
|
|
|
|
def RunTestIn(self, command_template, tmp):
|
|
tmp.Write(self.GetSource())
|
|
tmp.Close()
|
|
command = self.InstantiateTemplate(command_template, {
|
|
'path': tmp.name
|
|
})
|
|
(code, out, err) = self.Execute(command)
|
|
return TestResult(code, out, err, self)
|
|
|
|
def Run(self, command_template):
|
|
tmp = TempFile(suffix=".js", prefix="test262-", text=True)
|
|
try:
|
|
result = self.RunTestIn(command_template, tmp)
|
|
finally:
|
|
tmp.Dispose()
|
|
return result
|
|
|
|
def Print(self):
|
|
print self.GetSource()
|
|
|
|
def validate(self):
|
|
flags = self.testRecord.get("flags")
|
|
|
|
if not flags:
|
|
return
|
|
|
|
if 'raw' in flags:
|
|
if 'noStrict' in flags:
|
|
raise TypeError("The `raw` flag implies the `noStrict` flag")
|
|
elif 'onlyStrict' in flags:
|
|
raise TypeError(
|
|
"The `raw` flag is incompatible with the `onlyStrict` flag")
|
|
elif len(self.GetIncludeList()) > 0:
|
|
raise TypeError(
|
|
"The `raw` flag is incompatible with the `includes` tag")
|
|
|
|
class ProgressIndicator(object):
|
|
|
|
def __init__(self, count):
|
|
self.count = count
|
|
self.succeeded = 0
|
|
self.failed = 0
|
|
self.failed_tests = []
|
|
|
|
def HasRun(self, result):
|
|
result.ReportOutcome(True)
|
|
if result.HasUnexpectedOutcome():
|
|
self.failed += 1
|
|
self.failed_tests.append(result)
|
|
else:
|
|
self.succeeded += 1
|
|
|
|
|
|
def MakePlural(n):
|
|
if (n == 1):
|
|
return (n, "")
|
|
else:
|
|
return (n, "s")
|
|
|
|
def PercentFormat(partial, total):
|
|
return "%i test%s (%.1f%%)" % (MakePlural(partial) +
|
|
((100.0 * partial)/total,))
|
|
|
|
|
|
class TestSuite(object):
|
|
|
|
def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
|
|
# TODO: derive from packagerConfig.py
|
|
self.test_root = path.join(root, 'test')
|
|
self.lib_root = path.join(root, 'harness')
|
|
self.strict_only = strict_only
|
|
self.non_strict_only = non_strict_only
|
|
self.unmarked_default = unmarked_default
|
|
self.print_handle = print_handle
|
|
self.include_cache = { }
|
|
|
|
|
|
def Validate(self):
|
|
if not path.exists(self.test_root):
|
|
ReportError("No test repository found")
|
|
if not path.exists(self.lib_root):
|
|
ReportError("No test library found")
|
|
|
|
def IsHidden(self, path):
|
|
return path.startswith('.') or path == 'CVS'
|
|
|
|
def IsTestCase(self, path):
|
|
return path.endswith('.js')
|
|
|
|
def ShouldRun(self, rel_path, tests):
|
|
if len(tests) == 0:
|
|
return True
|
|
for test in tests:
|
|
if test in rel_path:
|
|
return True
|
|
return False
|
|
|
|
def GetInclude(self, name):
|
|
if not name in self.include_cache:
|
|
static = path.join(self.lib_root, name)
|
|
if path.exists(static):
|
|
f = open(static)
|
|
contents = stripHeader(f.read())
|
|
contents = re.sub(r'\r\n', '\n', contents)
|
|
self.include_cache[name] = contents + "\n"
|
|
f.close()
|
|
else:
|
|
ReportError("Can't find: " + static)
|
|
return self.include_cache[name]
|
|
|
|
def EnumerateTests(self, tests):
|
|
logging.info("Listing tests in %s", self.test_root)
|
|
cases = []
|
|
for root, dirs, files in os.walk(self.test_root):
|
|
for f in [x for x in dirs if self.IsHidden(x)]:
|
|
dirs.remove(f)
|
|
dirs.sort()
|
|
for f in sorted(files):
|
|
if self.IsTestCase(f):
|
|
full_path = path.join(root, f)
|
|
if full_path.startswith(self.test_root):
|
|
rel_path = full_path[len(self.test_root)+1:]
|
|
else:
|
|
logging.warning("Unexpected path %s", full_path)
|
|
rel_path = full_path
|
|
if self.ShouldRun(rel_path, tests):
|
|
basename = path.basename(full_path)[:-3]
|
|
name = rel_path.split(path.sep)[:-1] + [basename]
|
|
if EXCLUDE_LIST.count(basename) >= 1:
|
|
print 'Excluded: ' + basename
|
|
else:
|
|
if not self.non_strict_only:
|
|
strict_case = TestCase(self, name, full_path, True)
|
|
if not strict_case.IsNoStrict():
|
|
if strict_case.IsOnlyStrict() or \
|
|
self.unmarked_default in ['both', 'strict']:
|
|
cases.append(strict_case)
|
|
if not self.strict_only:
|
|
non_strict_case = TestCase(self, name, full_path, False)
|
|
if not non_strict_case.IsOnlyStrict():
|
|
if non_strict_case.IsNoStrict() or \
|
|
self.unmarked_default in ['both', 'non_strict']:
|
|
cases.append(non_strict_case)
|
|
logging.info("Done listing tests")
|
|
return cases
|
|
|
|
|
|
def PrintSummary(self, progress, logfile):
|
|
|
|
def write(s):
|
|
if logfile:
|
|
self.logf.write(s + "\n")
|
|
print s
|
|
|
|
print
|
|
write("=== Summary ===");
|
|
count = progress.count
|
|
succeeded = progress.succeeded
|
|
failed = progress.failed
|
|
write(" - Ran %i test%s" % MakePlural(count))
|
|
if progress.failed == 0:
|
|
write(" - All tests succeeded")
|
|
else:
|
|
write(" - Passed " + PercentFormat(succeeded, count))
|
|
write(" - Failed " + PercentFormat(failed, count))
|
|
positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
|
|
negative = [c for c in progress.failed_tests if c.case.IsNegative()]
|
|
if len(positive) > 0:
|
|
print
|
|
write("Failed Tests")
|
|
for result in positive:
|
|
write(" %s in %s" % (result.case.GetName(), result.case.GetMode()))
|
|
if len(negative) > 0:
|
|
print
|
|
write("Expected to fail but passed ---")
|
|
for result in negative:
|
|
write(" %s in %s" % (result.case.GetName(), result.case.GetMode()))
|
|
|
|
def PrintFailureOutput(self, progress, logfile):
|
|
for result in progress.failed_tests:
|
|
if logfile:
|
|
self.WriteLog(result)
|
|
print
|
|
result.ReportOutcome(False)
|
|
|
|
def Run(self, command_template, tests, print_summary, full_summary, logname, junitfile):
|
|
if not "{{path}}" in command_template:
|
|
command_template += " {{path}}"
|
|
cases = self.EnumerateTests(tests)
|
|
if len(cases) == 0:
|
|
ReportError("No tests to run")
|
|
progress = ProgressIndicator(len(cases))
|
|
if logname:
|
|
self.logf = open(logname, "w")
|
|
if junitfile:
|
|
self.outfile = open(junitfile, "w")
|
|
TestSuitesElement = xmlj.Element("testsuites")
|
|
TestSuiteElement = xmlj.Element("testsuite")
|
|
TestSuitesElement.append(TestSuiteElement)
|
|
TestSuiteElement.attrib["name "] = "test262"
|
|
for x in range(len(EXCLUDE_LIST)):
|
|
if self.ShouldRun (unicode(EXCLUDE_LIST[x].encode('utf-8','ignore')), tests):
|
|
SkipCaseElement = xmlj.Element("testcase")
|
|
SkipCaseElement.attrib["classname"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
|
|
SkipCaseElement.attrib["name"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
|
|
SkipElement = xmlj.Element("skipped")
|
|
SkipElement.attrib["message"] = unicode(EXCLUDE_REASON[x].firstChild.nodeValue)
|
|
SkipCaseElement.append(SkipElement)
|
|
TestSuiteElement.append(SkipCaseElement)
|
|
|
|
for case in cases:
|
|
result = case.Run(command_template)
|
|
if junitfile:
|
|
TestCaseElement = result.XmlAssemble(result)
|
|
TestSuiteElement.append(TestCaseElement)
|
|
if case == cases[len(cases)-1]:
|
|
xmlj.ElementTree(TestSuitesElement).write(junitfile, "UTF-8")
|
|
if logname:
|
|
self.WriteLog(result)
|
|
progress.HasRun(result)
|
|
|
|
if print_summary:
|
|
self.PrintSummary(progress, logname)
|
|
if full_summary:
|
|
self.PrintFailureOutput(progress, logname)
|
|
else:
|
|
print
|
|
print "Use --full-summary to see output from failed tests"
|
|
print
|
|
return progress.failed
|
|
|
|
def WriteLog(self, result):
|
|
name = result.case.GetName()
|
|
mode = result.case.GetMode()
|
|
if result.HasUnexpectedOutcome():
|
|
if result.case.IsNegative():
|
|
self.logf.write("=== %s was expected to fail in %s, but didn't === \n" % (name, mode))
|
|
self.logf.write("--- expected error: %s ---\n" % result.case.GetNegative())
|
|
result.WriteOutput(self.logf)
|
|
else:
|
|
self.logf.write("=== %s failed in %s === \n" % (name, mode))
|
|
result.WriteOutput(self.logf)
|
|
self.logf.write("===\n")
|
|
elif result.case.IsNegative():
|
|
self.logf.write("%s failed in %s as expected \n" % (name, mode))
|
|
else:
|
|
self.logf.write("%s passed in %s \n" % (name, mode))
|
|
|
|
def Print(self, tests):
|
|
cases = self.EnumerateTests(tests)
|
|
if len(cases) > 0:
|
|
cases[0].Print()
|
|
|
|
def ListIncludes(self, tests):
|
|
cases = self.EnumerateTests(tests)
|
|
includes_dict = Counter()
|
|
for case in cases:
|
|
includes = case.GetIncludeList()
|
|
includes_dict.update(includes)
|
|
|
|
print includes_dict
|
|
|
|
|
|
def Main():
|
|
code = 0
|
|
parser = BuildOptions()
|
|
(options, args) = parser.parse_args()
|
|
ValidateOptions(options)
|
|
test_suite = TestSuite(options.tests,
|
|
options.strict_only,
|
|
options.non_strict_only,
|
|
options.unmarked_default,
|
|
options.print_handle)
|
|
test_suite.Validate()
|
|
if options.loglevel == 'debug':
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
elif options.loglevel == 'info':
|
|
logging.basicConfig(level=logging.INFO)
|
|
elif options.loglevel == 'warning':
|
|
logging.basicConfig(level=logging.WARNING)
|
|
elif options.loglevel == 'error':
|
|
logging.basicConfig(level=logging.ERROR)
|
|
elif options.loglevel == 'critical':
|
|
logging.basicConfig(level=logging.CRITICAL)
|
|
if options.cat:
|
|
test_suite.Print(args)
|
|
elif options.list_includes:
|
|
test_suite.ListIncludes(args)
|
|
else:
|
|
code = test_suite.Run(options.command, args,
|
|
options.summary or options.full_summary,
|
|
options.full_summary,
|
|
options.logname,
|
|
options.junitname)
|
|
return code
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
code = Main()
|
|
sys.exit(code)
|
|
except Test262Error, e:
|
|
print "Error: %s" % e.message
|
|
sys.exit(1)
|