From 0ad1d9be4041c84b8b7a3b9bb23cc5e81fdc3ff1 Mon Sep 17 00:00:00 2001 From: Foo Date: Sat, 5 Aug 2017 13:53:24 +0300 Subject: [PATCH] More granular failure reports in summary from vterm tests --- tests/modules/lib/__init__.py | 84 ++++++++++++++++++++++++++++++++ tests/modules/lib/terminal.py | 30 ++++++------ tests/shlib/common.sh | 2 +- tests/test_in_vterm/test_tmux.py | 23 +++++---- tests/test_in_vterm/test_vim.py | 19 +++++--- 5 files changed, 125 insertions(+), 33 deletions(-) diff --git a/tests/modules/lib/__init__.py b/tests/modules/lib/__init__.py index 9a18faf2..b5a30d7d 100644 --- a/tests/modules/lib/__init__.py +++ b/tests/modules/lib/__init__.py @@ -3,6 +3,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_funct import imp import sys +import os class Pl(object): @@ -161,3 +162,86 @@ def replace_env(key, new, environ=None, **kwargs): r = kwargs.copy() r['environ'] = environ or {} return ItemReplace(r['environ'], key, new, r) + + +class PowerlineSingleTest(object): + def __init__(self, suite, name): + self.suite = suite + self.name = name + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is not None: + self.exception('Exception while running test: {0!r}'.format( + exc_value)) + + def fail(self, message, allow_failure=False): + return self.suite.fail(self.name, message, allow_failure) + + def exception(self, message, allow_failure=False): + return self.suite.exception(self.name, message, allow_failure) + + +class PowerlineDummyTest(object): + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def fail(self, *args, **kwargs): + pass + + def exception(self, *args, **kwargs): + pass + + +class PowerlineTestSuite(object): + def __init__(self, name): + self.name = name + + def __enter__(self): + self.saved_current_suite = os.environ['POWERLINE_CURRENT_SUITE'] + os.environ['POWERLINE_CURRENT_SUITE'] = ( + self.saved_current_suite + '/' + self.name) + self.suite = self.saved_current_suite + '/' + self.name + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is not None: + self.exception( + 'suite_noexcept', + 'Exception while running test suite: {0!r}'.format(exc_value), + ) + os.environ['POWERLINE_CURRENT_SUITE'] = self.saved_current_suite + + def record_test_failure(self, fail_char, test_name, message, allow_failure=False): + if allow_failure: + fail_char = 'A' + fail_char + full_msg = '{fail_char} {suite}|{test_name} :: {message}'.format( + fail_char=fail_char, + suite=self.suite, + test_name=test_name, + message=message, + ) + print('Failed: ' + full_msg) + with open(os.environ['FAILURES_FILE'], 'a') as ffd: + ffd.write(full_msg + '\n') + return False + + def exception(self, test_name, message, allow_failure=False): + return self.record_test_failure('E', test_name, message, allow_failure) + + def fail(self, test_name, message, allow_failure=False): + return self.record_test_failure('F', test_name, message, allow_failure) + + def test(self, name, attempts=0): + if not attempts: + return PowerlineSingleTest(self, name) + else: + return PowerlineDummyTest() + + def subsuite(self, name): + return PowerlineTestSuite(name) diff --git a/tests/modules/lib/terminal.py b/tests/modules/lib/terminal.py index 860b20e1..ab52b8de 100644 --- a/tests/modules/lib/terminal.py +++ b/tests/modules/lib/terminal.py @@ -220,7 +220,7 @@ def get_env(vterm_path, test_dir, *args, **kwargs): return env -def do_terminal_tests(tests, cmd, dim, args, env, cwd=None, fin_cb=None, +def do_terminal_tests(tests, cmd, dim, args, env, suite, cwd=None, fin_cb=None, last_attempt_cb=None, attempts=3): lib = os.environ.get('POWERLINE_LIBVTERM') if not lib: @@ -243,19 +243,21 @@ def do_terminal_tests(tests, cmd, dim, args, env, cwd=None, fin_cb=None, ret = True - for test in tests: - try: - test_prep = test['prep_cb'] - except KeyError: - pass - else: - test_prep(p) - ret = ( - ret - and test_expected_result(p, test, attempts == 0, - last_attempt_cb, - test.get('attempts', 3)) - ) + for i, test in enumerate(tests): + with suite.test(test.get('name', 'test_{0}'.format(i)), + attempts) as ptest: + try: + test_prep = test['prep_cb'] + except KeyError: + pass + else: + test_prep(p) + test_result = test_expected_result(p, test, attempts == 0, + last_attempt_cb, + test.get('attempts', 3)) + if not test_result: + ptest.fail('Result does not match expected') + ret = ret and test_result if ret: return ret diff --git a/tests/shlib/common.sh b/tests/shlib/common.sh index f16a1559..ed4504a9 100644 --- a/tests/shlib/common.sh +++ b/tests/shlib/common.sh @@ -13,7 +13,7 @@ if test -z "$FAILED" ; then FAIL_SUMMARY="" TMP_ROOT="$ROOT/tests/tmp" - FAILURES_FILE="$ROOT/tests/failures" + export FAILURES_FILE="$ROOT/tests/failures" fi ANSI_CLEAR="\033[0K" diff --git a/tests/test_in_vterm/test_tmux.py b/tests/test_in_vterm/test_tmux.py index 76b9dffd..dfccc24e 100755 --- a/tests/test_in_vterm/test_tmux.py +++ b/tests/test_in_vterm/test_tmux.py @@ -17,6 +17,7 @@ from powerline import get_fallback_logger from tests.modules.lib.terminal import (ExpectProcess, MutableDimensions, do_terminal_tests, get_env) +from tests.modules.lib import PowerlineTestSuite TEST_ROOT = os.path.abspath(os.environ['TEST_ROOT']) @@ -229,16 +230,18 @@ def main(attempts=3): 'new-window', 'bash --norc --noprofile -i', ';', ] - return do_terminal_tests( - tests=tests, - cmd=tmux_exe, - dim=dim, - args=args, - env=env, - cwd=TEST_ROOT, - fin_cb=tmux_fin_cb, - last_attempt_cb=print_tmux_logs, - ) + with PowerlineTestSuite('tmux') as suite: + return do_terminal_tests( + tests=tests, + cmd=tmux_exe, + dim=dim, + args=args, + env=env, + cwd=TEST_ROOT, + fin_cb=tmux_fin_cb, + last_attempt_cb=print_tmux_logs, + suite=suite, + ) if __name__ == '__main__': diff --git a/tests/test_in_vterm/test_vim.py b/tests/test_in_vterm/test_vim.py index 663d1302..11da223b 100755 --- a/tests/test_in_vterm/test_vim.py +++ b/tests/test_in_vterm/test_vim.py @@ -14,6 +14,7 @@ from powerline.lib.dict import updated from tests.modules.lib.terminal import (ExpectProcess, MutableDimensions, do_terminal_tests, get_env) +from tests.modules.lib import PowerlineTestSuite TEST_ROOT = os.path.abspath(os.environ['TEST_ROOT']) @@ -53,14 +54,16 @@ def main(attempts=3): tests = ( ) - return do_terminal_tests( - tests=tests, - cmd=vim_exe, - dim=dim, - args=args, - env=env, - cwd=TEST_ROOT, - ) + with PowerlineTestSuite('vim') as suite: + return do_terminal_tests( + tests=tests, + cmd=vim_exe, + dim=dim, + args=args, + env=env, + cwd=TEST_ROOT, + suite=suite, + ) if __name__ == '__main__':