From 64093f79d3560815b143596ab11428a367ea9499 Mon Sep 17 00:00:00 2001 From: smikes Date: Tue, 11 Nov 2014 11:20:57 -0700 Subject: [PATCH] add some unit tests add test to repro #114 fix issue #114 factor out PercentFormat function new fn write; start factoring summary output fix tests to match consolidate "Failed Tests" message add test for successful run consolidate all_succeeded message consolidate summary line consolidate ntests failing negative tests squeeze whitespace --- tools/packaging/test/README.md | 2 +- tools/packaging/test/test_test262.py | 274 +++++++++++++++++++++++++++ tools/packaging/test262.py | 49 ++--- 3 files changed, 296 insertions(+), 29 deletions(-) create mode 100644 tools/packaging/test/test_test262.py diff --git a/tools/packaging/test/README.md b/tools/packaging/test/README.md index b28f77f4fd..6084f93b8e 100644 --- a/tools/packaging/test/README.md +++ b/tools/packaging/test/README.md @@ -6,6 +6,6 @@ This directory holds tests for the python code, not tests of EMCAScript ```` $ cd tools/packaging/test -$ python test*.py +$ for x in test*.py; do python $x; done ```` diff --git a/tools/packaging/test/test_test262.py b/tools/packaging/test/test_test262.py new file mode 100644 index 0000000000..fbd3dd4116 --- /dev/null +++ b/tools/packaging/test/test_test262.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python + +# Copyright 2014 by Sam Mikes. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import unittest + +import sys +import os +import cStringIO +from functools import wraps + +sys.path.insert(0, "..") + +import test262 + +class TestTest262(unittest.TestCase): + + def test_that_tests_run(self): + self.assertEqual(1 + 2, 3) + +class MockTest(object): + + def __init__(self, name, negative): + self.name = name + self.negative = negative if negative else False + self.strict_mode = False + + def GetName(self): + return self.name + + def IsNegative(self): + return self.negative + + def GetMode(self): + if self.strict_mode: + return "strict mode" + + return "non-strict mode" + +class MockResult(object): + + def __init__(self, case): + self.case = case + + + +class TestTestSuite(unittest.TestCase): + + def test_that_tests_run(self): + self.assertEqual(1 + 2, 3) + + def test_create_test_suite(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + self.assertNotEqual(test_suite, None) + + def test_summary(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + + progress = test262.ProgressIndicator(100) + progress.succeeded = 98 + progress.failed = 2 + + result = mute(True)(test_suite.PrintSummary)(progress, None) + self.assertEqual(""" +=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) +""", result) + + def test_summary_logfile(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + + progress = test262.ProgressIndicator(100) + progress.succeeded = 98 + progress.failed = 2 + + fake_log = cStringIO.StringIO() + test_suite.logf = fake_log + + result = mute(True)(test_suite.PrintSummary)(progress, True) + + expected_out = """ +=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) +""" + + expected_log = """=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) +""" + self.assertEqual(expected_out, result) + self.assertEqual(expected_log, fake_log.getvalue()) + + + def test_summary_withfails(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + + progress = test262.ProgressIndicator(100) + progress.succeeded = 98 + progress.failed = 2 + progress.failed_tests = [ + MockResult(MockTest("foo", False)), + MockResult(MockTest("bar", True)) + ] + + result = mute(True)(test_suite.PrintSummary)(progress, None) + self.assertEqual(""" +=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) + +Failed Tests + foo in non-strict mode + +Expected to fail but passed --- + bar in non-strict mode +""", result) + + + def test_summary_withfails_andlog(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + + progress = test262.ProgressIndicator(100) + progress.succeeded = 98 + progress.failed = 2 + progress.failed_tests = [ + MockResult(MockTest("foo", False)), + MockResult(MockTest("bar", True)) + ] + + fake_log = cStringIO.StringIO() + test_suite.logf = fake_log + + expected_out = """ +=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) + +Failed Tests + foo in non-strict mode + +Expected to fail but passed --- + bar in non-strict mode +""" + expected_log = """=== Summary === + - Ran 100 tests + - Passed 98 tests (98.0%) + - Failed 2 tests (2.0%) +Failed Tests + foo in non-strict mode +Expected to fail but passed --- + bar in non-strict mode +""" + + result = mute(True)(test_suite.PrintSummary)(progress, True) + self.assertEqual(expected_out, result) + self.assertEqual(expected_log, fake_log.getvalue()) + + + def test_summary_success_logfile(self): + test_suite = test262.TestSuite(".", + False, + False, + False, + None) + + progress = test262.ProgressIndicator(100) + progress.succeeded = 100 + progress.failed = 0 + + fake_log = cStringIO.StringIO() + test_suite.logf = fake_log + + result = mute(True)(test_suite.PrintSummary)(progress, True) + + expected_out = """ +=== Summary === + - Ran 100 tests + - All tests succeeded +""" + + expected_log = """=== Summary === + - Ran 100 tests + - All tests succeeded +""" + self.assertEqual(expected_out, result) + self.assertEqual(expected_log, fake_log.getvalue()) + + + def test_percent_format(self): + self.assertEqual(test262.PercentFormat(1, 100), "1 test (1.0%)") + self.assertEqual(test262.PercentFormat(0, 100), "0 tests (0.0%)") + self.assertEqual(test262.PercentFormat(99, 100), "99 tests (99.0%)") + + +# module level utility functions +# copied from https://stackoverflow.com/questions/2828953/silence-the-stdout-of-a-function-in-python-without-trashing-sys-stdout-and-resto + + +def mute(returns_output=False): + """ + Decorate a function that prints to stdout, intercepting the output. + If "returns_output" is True, the function will return a generator + yielding the printed lines instead of the return values. + + The decorator litterally hijack sys.stdout during each function + execution for ALL THE THREADS, so be careful with what you apply it to + and in which context. + + >>> def numbers(): + print "42" + print "1984" + ... + >>> numbers() + 42 + 1984 + >>> mute()(numbers)() + >>> list(mute(True)(numbers)()) + ['42', '1984'] + + """ + + def decorator(func): + + @wraps(func) + def wrapper(*args, **kwargs): + + saved_stdout = sys.stdout + sys.stdout = cStringIO.StringIO() + + try: + out = func(*args, **kwargs) + if returns_output: + out = sys.stdout.getvalue() + finally: + sys.stdout = saved_stdout + + return out + + return wrapper + + return decorator + + +if __name__ == '__main__': + unittest.main() + diff --git a/tools/packaging/test262.py b/tools/packaging/test262.py index aa396cbe10..40c756fc95 100755 --- a/tools/packaging/test262.py +++ b/tools/packaging/test262.py @@ -382,6 +382,11 @@ def MakePlural(n): else: return (n, "s") +def PercentFormat(partial, total): + return "%i test%s (%.1f%%)" % (MakePlural(partial) + + ((100.0 * partial)/total,)) + + class TestSuite(object): def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle): @@ -464,49 +469,37 @@ class TestSuite(object): logging.info("Done listing tests") return cases + def PrintSummary(self, progress, logfile): + + def write(s): + if logfile: + self.logf.write(s + "\n") + print s + print - if logfile: - self.logf.write("=== Summary === \n") - print "=== Summary ===" + write("=== Summary ==="); count = progress.count succeeded = progress.succeeded failed = progress.failed - if logfile: - self.logf.write(" - Ran %i test%s \n" % MakePlural(count)) - print " - Ran %i test%s" % MakePlural(count) + write(" - Ran %i test%s" % MakePlural(count)) if progress.failed == 0: - if logfile: - self.logf.write(" - All tests succeeded \n") - print " - All tests succeeded" - + write(" - All tests succeeded") else: - percent = ((100.0 * succeeded) / count,) - if logfile: - self.logf.write(" - Passed %i test%s (%.1f%%)\n" % (MakePlural(succeeded) + percent)) - print " - Passed %i test%s (%.1f%%)" % (MakePlural(succeeded) + percent) - percent = ((100.0 * failed) / count,) - if logfile: - self.logf.write(" - Failed %i test%s (%.1f%%) \n" % (MakePlural(failed) + percent)) - print " - Failed %i test%s (%.1f%%)" % (MakePlural(failed) + percent) + write(" - Passed " + PercentFormat(succeeded, count)) + write(" - Failed " + PercentFormat(failed, count)) positive = [c for c in progress.failed_tests if not c.case.IsNegative()] negative = [c for c in progress.failed_tests if c.case.IsNegative()] if len(positive) > 0: print - if logfile: - self.logf.write("Failed Tests \n") - print "Failed tests" + write("Failed Tests") for result in positive: - if logfile: - self.logf.write(" %s in %s \n" % (result.case.GetName(), result.case.GetMode())) - print " %s in %s" % (result.case.GetName(), result.case.GetMode()) + write(" %s in %s" % (result.case.GetName(), result.case.GetMode())) if len(negative) > 0: print - print "Expected to fail but passed ---" + write("Expected to fail but passed ---") for result in negative: - if logfile: - self.logfile.append(" %s in %s \n" % (result.case.GetName(), result.case.GetMode())) - print " %s in %s" % (result.case.GetName(), result.case.GetMode()) + write(" %s in %s" % (result.case.GetName(), result.case.GetMode())) def PrintFailureOutput(self, progress, logfile): for result in progress.failed_tests: