mirror of
https://github.com/tc39/test262.git
synced 2025-11-04 05:33:50 +01:00
7298a38552
sourceRevisionAtLastExport: 33f2fb0e53d135f0ee17cfccd9d993eb2a6f47de targetRevisionAtLastExport: 31340cbd9add103f586d501b0c3354b7b182abc0
309 lines
10 KiB
Python
309 lines
10 KiB
Python
# Copyright 2008 the V8 project authors. All rights reserved.
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following
|
|
# disclaimer in the documentation and/or other materials provided
|
|
# with the distribution.
|
|
# * Neither the name of Google Inc. nor the names of its
|
|
# contributors may be used to endorse or promote products derived
|
|
# from this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
from collections import OrderedDict
|
|
import itertools
|
|
import os
|
|
import re
|
|
|
|
from testrunner.local import statusfile
|
|
from testrunner.local import testsuite
|
|
from testrunner.objects import testcase
|
|
from testrunner.outproc import base as outproc
|
|
|
|
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
|
|
ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
|
|
SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
|
|
MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
|
|
NO_HARNESS_PATTERN = re.compile(r"^// NO HARNESS$", flags=re.MULTILINE)
|
|
|
|
|
|
# Flags known to misbehave when combining arbitrary mjsunit tests. Can also
|
|
# be compiled regular expressions.
|
|
COMBINE_TESTS_FLAGS_BLACKLIST = [
|
|
'--check-handle-count',
|
|
'--enable-tracing',
|
|
re.compile('--experimental.*'),
|
|
'--expose-trigger-failure',
|
|
re.compile('--harmony.*'),
|
|
'--mock-arraybuffer-allocator',
|
|
'--print-ast',
|
|
re.compile('--trace.*'),
|
|
'--wasm-lazy-compilation',
|
|
]
|
|
|
|
class TestSuite(testsuite.TestSuite):
|
|
def ListTests(self):
|
|
tests = []
|
|
for dirname, dirs, files in os.walk(self.root, followlinks=True):
|
|
for dotted in [x for x in dirs if x.startswith('.')]:
|
|
dirs.remove(dotted)
|
|
dirs.sort()
|
|
files.sort()
|
|
for filename in files:
|
|
if (filename.endswith(".js") and
|
|
filename != "mjsunit.js" and
|
|
filename != "mjsunit_suppressions.js"):
|
|
fullpath = os.path.join(dirname, filename)
|
|
relpath = fullpath[len(self.root) + 1 : -3]
|
|
testname = relpath.replace(os.path.sep, "/")
|
|
test = self._create_test(testname)
|
|
tests.append(test)
|
|
return tests
|
|
|
|
def _test_combiner_class(self):
|
|
return TestCombiner
|
|
|
|
def _test_class(self):
|
|
return TestCase
|
|
|
|
def _suppressed_test_class(self):
|
|
return SuppressedTestCase
|
|
|
|
|
|
class TestCase(testcase.D8TestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
super(TestCase, self).__init__(*args, **kwargs)
|
|
|
|
source = self.get_source()
|
|
|
|
files_list = [] # List of file names to append to command arguments.
|
|
files_match = FILES_PATTERN.search(source);
|
|
# Accept several lines of 'Files:'.
|
|
while True:
|
|
if files_match:
|
|
files_list += files_match.group(1).strip().split()
|
|
files_match = FILES_PATTERN.search(source, files_match.end())
|
|
else:
|
|
break
|
|
files = [ os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
|
|
for f in files_list ]
|
|
testfilename = os.path.join(self.suite.root,
|
|
self.path + self._get_suffix())
|
|
if SELF_SCRIPT_PATTERN.search(source):
|
|
files = (
|
|
["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] +
|
|
files)
|
|
|
|
if NO_HARNESS_PATTERN.search(source):
|
|
mjsunit_files = []
|
|
else:
|
|
mjsunit_files = [os.path.join(self.suite.root, "mjsunit.js")]
|
|
|
|
files_suffix = []
|
|
if MODULE_PATTERN.search(source):
|
|
files_suffix.append("--module")
|
|
files_suffix.append(testfilename)
|
|
|
|
self._source_files = files
|
|
self._source_flags = self._parse_source_flags(source)
|
|
self._mjsunit_files = mjsunit_files
|
|
self._files_suffix = files_suffix
|
|
self._env = self._parse_source_env(source)
|
|
|
|
def _parse_source_env(self, source):
|
|
env_match = ENV_PATTERN.search(source)
|
|
env = {}
|
|
if env_match:
|
|
for env_pair in env_match.group(1).strip().split():
|
|
var, value = env_pair.split('=')
|
|
env[var] = value
|
|
return env
|
|
|
|
def _get_source_flags(self):
|
|
return self._source_flags
|
|
|
|
def _get_files_params(self):
|
|
files = list(self._source_files)
|
|
if not self._test_config.no_harness:
|
|
files += self._mjsunit_files
|
|
files += self._files_suffix
|
|
if self._test_config.isolates:
|
|
files += ['--isolate'] + files
|
|
|
|
return files
|
|
|
|
def _get_cmd_env(self):
|
|
return self._env
|
|
|
|
def _get_source_path(self):
|
|
return os.path.join(self.suite.root, self.path + self._get_suffix())
|
|
|
|
|
|
class TestCombiner(testsuite.TestCombiner):
|
|
def get_group_key(self, test):
|
|
"""Combine tests with the same set of flags.
|
|
Ignore:
|
|
1. Some special cases where it's not obvious what to pass in the command.
|
|
2. Tests with flags that can cause failure even inside try-catch wrapper.
|
|
3. Tests that use async functions. Async functions can be scheduled after
|
|
exiting from try-catch wrapper and cause failure.
|
|
"""
|
|
if (len(test._files_suffix) > 1 or
|
|
test._env or
|
|
not test._mjsunit_files or
|
|
test._source_files):
|
|
return None
|
|
|
|
source_flags = test._get_source_flags()
|
|
if ('--expose-trigger-failure' in source_flags or
|
|
'--throws' in source_flags):
|
|
return None
|
|
|
|
source_code = test.get_source()
|
|
# Maybe we could just update the tests to await all async functions they
|
|
# call?
|
|
if 'async' in source_code:
|
|
return None
|
|
|
|
# TODO(machenbach): Remove grouping if combining tests in a flag-independent
|
|
# way works well.
|
|
return 1
|
|
|
|
def _combined_test_class(self):
|
|
return CombinedTest
|
|
|
|
|
|
class CombinedTest(testcase.D8TestCase):
|
|
"""Behaves like normal mjsunit tests except:
|
|
1. Expected outcome is always PASS
|
|
2. Instead of one file there is a try-catch wrapper with all combined tests
|
|
passed as arguments.
|
|
"""
|
|
def __init__(self, name, tests):
|
|
super(CombinedTest, self).__init__(tests[0].suite, '', name,
|
|
tests[0]._test_config)
|
|
self._tests = tests
|
|
|
|
def _prepare_outcomes(self, force_update=True):
|
|
self._statusfile_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
|
|
self.expected_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
|
|
|
|
def _get_shell_flags(self):
|
|
"""In addition to standard set of shell flags it appends:
|
|
--disable-abortjs: %AbortJS can abort the test even inside
|
|
trycatch-wrapper, so we disable it.
|
|
--es-staging: We blacklist all harmony flags due to false positives,
|
|
but always pass the staging flag to cover the mature features.
|
|
--omit-quit: Calling quit() in JS would otherwise early terminate.
|
|
--quiet-load: suppress any stdout from load() function used by
|
|
trycatch-wrapper.
|
|
"""
|
|
return [
|
|
'--test',
|
|
'--disable-abortjs',
|
|
'--es-staging',
|
|
'--omit-quit',
|
|
'--quiet-load',
|
|
]
|
|
|
|
def _get_cmd_params(self):
|
|
return (
|
|
super(CombinedTest, self)._get_cmd_params() +
|
|
['tools/testrunner/trycatch_loader.js', '--'] +
|
|
self._tests[0]._mjsunit_files +
|
|
['--'] +
|
|
[t._files_suffix[0] for t in self._tests]
|
|
)
|
|
|
|
def _merge_flags(self, flags):
|
|
"""Merges flags from a list of flags.
|
|
|
|
Flag values not starting with '-' are merged with the preceeding flag,
|
|
e.g. --foo 1 will become --foo=1. All other flags remain the same.
|
|
|
|
Returns: A generator of flags.
|
|
"""
|
|
if not flags:
|
|
return
|
|
# Iterate over flag pairs. ['-'] is a sentinel value for the last iteration.
|
|
for flag1, flag2 in itertools.izip(flags, flags[1:] + ['-']):
|
|
if not flag2.startswith('-'):
|
|
assert '=' not in flag1
|
|
yield flag1 + '=' + flag2
|
|
elif flag1.startswith('-'):
|
|
yield flag1
|
|
|
|
def _is_flag_blacklisted(self, flag):
|
|
for item in COMBINE_TESTS_FLAGS_BLACKLIST:
|
|
if isinstance(item, basestring):
|
|
if item == flag:
|
|
return True
|
|
elif item.match(flag):
|
|
return True
|
|
return False
|
|
|
|
def _get_combined_flags(self, flags_gen):
|
|
"""Combines all flags - dedupes, keeps order and filters some flags.
|
|
|
|
Args:
|
|
flags_gen: Generator for flag lists.
|
|
Returns: A list of flags.
|
|
"""
|
|
merged_flags = self._merge_flags(list(itertools.chain(*flags_gen)))
|
|
unique_flags = OrderedDict((flag, True) for flag in merged_flags).keys()
|
|
return [
|
|
flag for flag in unique_flags
|
|
if not self._is_flag_blacklisted(flag)
|
|
]
|
|
|
|
def _get_source_flags(self):
|
|
# Combine flags from all source files.
|
|
return self._get_combined_flags(
|
|
test._get_source_flags() for test in self._tests)
|
|
|
|
def _get_statusfile_flags(self):
|
|
# Combine flags from all status file entries.
|
|
return self._get_combined_flags(
|
|
test._get_statusfile_flags() for test in self._tests)
|
|
|
|
|
|
class SuppressedTestCase(TestCase):
|
|
"""The same as a standard mjsunit test case with all asserts as no-ops."""
|
|
def __init__(self, *args, **kwargs):
|
|
super(SuppressedTestCase, self).__init__(*args, **kwargs)
|
|
self._mjsunit_files.append(
|
|
os.path.join(self.suite.root, "mjsunit_suppressions.js"))
|
|
|
|
def _prepare_outcomes(self, *args, **kwargs):
|
|
super(SuppressedTestCase, self)._prepare_outcomes(*args, **kwargs)
|
|
# Skip tests expected to fail. We suppress all asserts anyways, but some
|
|
# tests are expected to fail with type errors or even dchecks, and we
|
|
# can't differentiate that.
|
|
if statusfile.FAIL in self._statusfile_outcomes:
|
|
self._statusfile_outcomes = [statusfile.SKIP]
|
|
|
|
def _get_extra_flags(self, *args, **kwargs):
|
|
return (
|
|
super(SuppressedTestCase, self)._get_extra_flags(*args, **kwargs) +
|
|
['--disable-abortjs']
|
|
)
|
|
|
|
|
|
def GetSuite(*args, **kwargs):
|
|
return TestSuite(*args, **kwargs)
|