mirror of https://github.com/tc39/test262.git
add some unit tests
add test to repro #114 fix issue #114 factor out PercentFormat function new fn write; start factoring summary output fix tests to match consolidate "Failed Tests" message add test for successful run consolidate all_succeeded message consolidate summary line consolidate ntests failing negative tests squeeze whitespace
This commit is contained in:
smikes
parent
61113dbfb3
commit
64093f79d3
|
|
@ -6,6 +6,6 @@ This directory holds tests for the python code, not tests of EMCAScript
|
|||
|
||||
````
|
||||
$ cd tools/packaging/test
|
||||
$ python test*.py
|
||||
$ for x in test*.py; do python $x; done
|
||||
````
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2014 by Sam Mikes. All rights reserved.
|
||||
# This code is governed by the BSD license found in the LICENSE file.
|
||||
|
||||
import unittest
|
||||
|
||||
import sys
|
||||
import os
|
||||
import cStringIO
|
||||
from functools import wraps
|
||||
|
||||
sys.path.insert(0, "..")
|
||||
|
||||
import test262
|
||||
|
||||
class TestTest262(unittest.TestCase):
|
||||
|
||||
def test_that_tests_run(self):
|
||||
self.assertEqual(1 + 2, 3)
|
||||
|
||||
class MockTest(object):
|
||||
|
||||
def __init__(self, name, negative):
|
||||
self.name = name
|
||||
self.negative = negative if negative else False
|
||||
self.strict_mode = False
|
||||
|
||||
def GetName(self):
|
||||
return self.name
|
||||
|
||||
def IsNegative(self):
|
||||
return self.negative
|
||||
|
||||
def GetMode(self):
|
||||
if self.strict_mode:
|
||||
return "strict mode"
|
||||
|
||||
return "non-strict mode"
|
||||
|
||||
class MockResult(object):
|
||||
|
||||
def __init__(self, case):
|
||||
self.case = case
|
||||
|
||||
|
||||
|
||||
class TestTestSuite(unittest.TestCase):
|
||||
|
||||
def test_that_tests_run(self):
|
||||
self.assertEqual(1 + 2, 3)
|
||||
|
||||
def test_create_test_suite(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
self.assertNotEqual(test_suite, None)
|
||||
|
||||
def test_summary(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
|
||||
progress = test262.ProgressIndicator(100)
|
||||
progress.succeeded = 98
|
||||
progress.failed = 2
|
||||
|
||||
result = mute(True)(test_suite.PrintSummary)(progress, None)
|
||||
self.assertEqual("""
|
||||
=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
""", result)
|
||||
|
||||
def test_summary_logfile(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
|
||||
progress = test262.ProgressIndicator(100)
|
||||
progress.succeeded = 98
|
||||
progress.failed = 2
|
||||
|
||||
fake_log = cStringIO.StringIO()
|
||||
test_suite.logf = fake_log
|
||||
|
||||
result = mute(True)(test_suite.PrintSummary)(progress, True)
|
||||
|
||||
expected_out = """
|
||||
=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
"""
|
||||
|
||||
expected_log = """=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
"""
|
||||
self.assertEqual(expected_out, result)
|
||||
self.assertEqual(expected_log, fake_log.getvalue())
|
||||
|
||||
|
||||
def test_summary_withfails(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
|
||||
progress = test262.ProgressIndicator(100)
|
||||
progress.succeeded = 98
|
||||
progress.failed = 2
|
||||
progress.failed_tests = [
|
||||
MockResult(MockTest("foo", False)),
|
||||
MockResult(MockTest("bar", True))
|
||||
]
|
||||
|
||||
result = mute(True)(test_suite.PrintSummary)(progress, None)
|
||||
self.assertEqual("""
|
||||
=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
|
||||
Failed Tests
|
||||
foo in non-strict mode
|
||||
|
||||
Expected to fail but passed ---
|
||||
bar in non-strict mode
|
||||
""", result)
|
||||
|
||||
|
||||
def test_summary_withfails_andlog(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
|
||||
progress = test262.ProgressIndicator(100)
|
||||
progress.succeeded = 98
|
||||
progress.failed = 2
|
||||
progress.failed_tests = [
|
||||
MockResult(MockTest("foo", False)),
|
||||
MockResult(MockTest("bar", True))
|
||||
]
|
||||
|
||||
fake_log = cStringIO.StringIO()
|
||||
test_suite.logf = fake_log
|
||||
|
||||
expected_out = """
|
||||
=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
|
||||
Failed Tests
|
||||
foo in non-strict mode
|
||||
|
||||
Expected to fail but passed ---
|
||||
bar in non-strict mode
|
||||
"""
|
||||
expected_log = """=== Summary ===
|
||||
- Ran 100 tests
|
||||
- Passed 98 tests (98.0%)
|
||||
- Failed 2 tests (2.0%)
|
||||
Failed Tests
|
||||
foo in non-strict mode
|
||||
Expected to fail but passed ---
|
||||
bar in non-strict mode
|
||||
"""
|
||||
|
||||
result = mute(True)(test_suite.PrintSummary)(progress, True)
|
||||
self.assertEqual(expected_out, result)
|
||||
self.assertEqual(expected_log, fake_log.getvalue())
|
||||
|
||||
|
||||
def test_summary_success_logfile(self):
|
||||
test_suite = test262.TestSuite(".",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
None)
|
||||
|
||||
progress = test262.ProgressIndicator(100)
|
||||
progress.succeeded = 100
|
||||
progress.failed = 0
|
||||
|
||||
fake_log = cStringIO.StringIO()
|
||||
test_suite.logf = fake_log
|
||||
|
||||
result = mute(True)(test_suite.PrintSummary)(progress, True)
|
||||
|
||||
expected_out = """
|
||||
=== Summary ===
|
||||
- Ran 100 tests
|
||||
- All tests succeeded
|
||||
"""
|
||||
|
||||
expected_log = """=== Summary ===
|
||||
- Ran 100 tests
|
||||
- All tests succeeded
|
||||
"""
|
||||
self.assertEqual(expected_out, result)
|
||||
self.assertEqual(expected_log, fake_log.getvalue())
|
||||
|
||||
|
||||
def test_percent_format(self):
|
||||
self.assertEqual(test262.PercentFormat(1, 100), "1 test (1.0%)")
|
||||
self.assertEqual(test262.PercentFormat(0, 100), "0 tests (0.0%)")
|
||||
self.assertEqual(test262.PercentFormat(99, 100), "99 tests (99.0%)")
|
||||
|
||||
|
||||
# module level utility functions
|
||||
# copied from https://stackoverflow.com/questions/2828953/silence-the-stdout-of-a-function-in-python-without-trashing-sys-stdout-and-resto
|
||||
|
||||
|
||||
def mute(returns_output=False):
|
||||
"""
|
||||
Decorate a function that prints to stdout, intercepting the output.
|
||||
If "returns_output" is True, the function will return a generator
|
||||
yielding the printed lines instead of the return values.
|
||||
|
||||
The decorator litterally hijack sys.stdout during each function
|
||||
execution for ALL THE THREADS, so be careful with what you apply it to
|
||||
and in which context.
|
||||
|
||||
>>> def numbers():
|
||||
print "42"
|
||||
print "1984"
|
||||
...
|
||||
>>> numbers()
|
||||
42
|
||||
1984
|
||||
>>> mute()(numbers)()
|
||||
>>> list(mute(True)(numbers)())
|
||||
['42', '1984']
|
||||
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
|
||||
saved_stdout = sys.stdout
|
||||
sys.stdout = cStringIO.StringIO()
|
||||
|
||||
try:
|
||||
out = func(*args, **kwargs)
|
||||
if returns_output:
|
||||
out = sys.stdout.getvalue()
|
||||
finally:
|
||||
sys.stdout = saved_stdout
|
||||
|
||||
return out
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
|
|
@ -382,6 +382,11 @@ def MakePlural(n):
|
|||
else:
|
||||
return (n, "s")
|
||||
|
||||
def PercentFormat(partial, total):
|
||||
return "%i test%s (%.1f%%)" % (MakePlural(partial) +
|
||||
((100.0 * partial)/total,))
|
||||
|
||||
|
||||
class TestSuite(object):
|
||||
|
||||
def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
|
||||
|
|
@ -464,49 +469,37 @@ class TestSuite(object):
|
|||
logging.info("Done listing tests")
|
||||
return cases
|
||||
|
||||
|
||||
def PrintSummary(self, progress, logfile):
|
||||
|
||||
def write(s):
|
||||
if logfile:
|
||||
self.logf.write(s + "\n")
|
||||
print s
|
||||
|
||||
print
|
||||
if logfile:
|
||||
self.logf.write("=== Summary === \n")
|
||||
print "=== Summary ==="
|
||||
write("=== Summary ===");
|
||||
count = progress.count
|
||||
succeeded = progress.succeeded
|
||||
failed = progress.failed
|
||||
if logfile:
|
||||
self.logf.write(" - Ran %i test%s \n" % MakePlural(count))
|
||||
print " - Ran %i test%s" % MakePlural(count)
|
||||
write(" - Ran %i test%s" % MakePlural(count))
|
||||
if progress.failed == 0:
|
||||
if logfile:
|
||||
self.logf.write(" - All tests succeeded \n")
|
||||
print " - All tests succeeded"
|
||||
|
||||
write(" - All tests succeeded")
|
||||
else:
|
||||
percent = ((100.0 * succeeded) / count,)
|
||||
if logfile:
|
||||
self.logf.write(" - Passed %i test%s (%.1f%%)\n" % (MakePlural(succeeded) + percent))
|
||||
print " - Passed %i test%s (%.1f%%)" % (MakePlural(succeeded) + percent)
|
||||
percent = ((100.0 * failed) / count,)
|
||||
if logfile:
|
||||
self.logf.write(" - Failed %i test%s (%.1f%%) \n" % (MakePlural(failed) + percent))
|
||||
print " - Failed %i test%s (%.1f%%)" % (MakePlural(failed) + percent)
|
||||
write(" - Passed " + PercentFormat(succeeded, count))
|
||||
write(" - Failed " + PercentFormat(failed, count))
|
||||
positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
|
||||
negative = [c for c in progress.failed_tests if c.case.IsNegative()]
|
||||
if len(positive) > 0:
|
||||
print
|
||||
if logfile:
|
||||
self.logf.write("Failed Tests \n")
|
||||
print "Failed tests"
|
||||
write("Failed Tests")
|
||||
for result in positive:
|
||||
if logfile:
|
||||
self.logf.write(" %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
|
||||
print " %s in %s" % (result.case.GetName(), result.case.GetMode())
|
||||
write(" %s in %s" % (result.case.GetName(), result.case.GetMode()))
|
||||
if len(negative) > 0:
|
||||
print
|
||||
print "Expected to fail but passed ---"
|
||||
write("Expected to fail but passed ---")
|
||||
for result in negative:
|
||||
if logfile:
|
||||
self.logfile.append(" %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
|
||||
print " %s in %s" % (result.case.GetName(), result.case.GetMode())
|
||||
write(" %s in %s" % (result.case.GetName(), result.case.GetMode()))
|
||||
|
||||
def PrintFailureOutput(self, progress, logfile):
|
||||
for result in progress.failed_tests:
|
||||
|
|
|
|||
Loading…
Reference in New Issue