More granular failure reports in summary from vterm tests

tests/modules/lib/__init__.py

@@ -3,6 +3,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_funct
 
 import imp
 import sys
+import os
 
 
 class Pl(object):
@@ -161,3 +162,86 @@ def replace_env(key, new, environ=None, **kwargs):
 	r = kwargs.copy()
 	r['environ'] = environ or {}
 	return ItemReplace(r['environ'], key, new, r)
+
+
+class PowerlineSingleTest(object):
+	def __init__(self, suite, name):
+		self.suite = suite
+		self.name = name
+
+	def __enter__(self):
+		return self
+
+	def __exit__(self, exc_type, exc_value, traceback):
+		if exc_type is not None:
+			self.exception('Exception while running test: {0!r}'.format(
+				exc_value))
+
+	def fail(self, message, allow_failure=False):
+		return self.suite.fail(self.name, message, allow_failure)
+
+	def exception(self, message, allow_failure=False):
+		return self.suite.exception(self.name, message, allow_failure)
+
+
+class PowerlineDummyTest(object):
+	def __enter__(self):
+		return self
+
+	def __exit__(self, *args):
+		pass
+
+	def fail(self, *args, **kwargs):
+		pass
+
+	def exception(self, *args, **kwargs):
+		pass
+
+
+class PowerlineTestSuite(object):
+	def __init__(self, name):
+		self.name = name
+
+	def __enter__(self):
+		self.saved_current_suite = os.environ['POWERLINE_CURRENT_SUITE']
+		os.environ['POWERLINE_CURRENT_SUITE'] = (
+			self.saved_current_suite + '/' + self.name)
+		self.suite = self.saved_current_suite + '/' + self.name
+		return self
+
+	def __exit__(self, exc_type, exc_value, traceback):
+		if exc_type is not None:
+			self.exception(
+				'suite_noexcept',
+				'Exception while running test suite: {0!r}'.format(exc_value),
+			)
+		os.environ['POWERLINE_CURRENT_SUITE'] = self.saved_current_suite
+
+	def record_test_failure(self, fail_char, test_name, message, allow_failure=False):
+		if allow_failure:
+			fail_char = 'A' + fail_char
+		full_msg = '{fail_char} {suite}|{test_name} :: {message}'.format(
+			fail_char=fail_char,
+			suite=self.suite,
+			test_name=test_name,
+			message=message,
+		)
+		print('Failed: ' + full_msg)
+		with open(os.environ['FAILURES_FILE'], 'a') as ffd:
+			ffd.write(full_msg + '\n')
+		return False
+
+	def exception(self, test_name, message, allow_failure=False):
+		return self.record_test_failure('E', test_name, message, allow_failure)
+
+	def fail(self, test_name, message, allow_failure=False):
+		return self.record_test_failure('F', test_name, message, allow_failure)
+
+	def test(self, name, attempts=0):
+		if not attempts:
+			return PowerlineSingleTest(self, name)
+		else:
+			return PowerlineDummyTest()
+
+	def subsuite(self, name):
+		return PowerlineTestSuite(name)

tests/modules/lib/terminal.py

@@ -220,7 +220,7 @@ def get_env(vterm_path, test_dir, *args, **kwargs):
 	return env
 
 
-def do_terminal_tests(tests, cmd, dim, args, env, cwd=None, fin_cb=None,
+def do_terminal_tests(tests, cmd, dim, args, env, suite, cwd=None, fin_cb=None,
                       last_attempt_cb=None, attempts=3):
 	lib = os.environ.get('POWERLINE_LIBVTERM')
 	if not lib:
@@ -243,19 +243,21 @@ def do_terminal_tests(tests, cmd, dim, args, env, cwd=None, fin_cb=None,
 
 			ret = True
 
-			for test in tests:
-				try:
-					test_prep = test['prep_cb']
-				except KeyError:
-					pass
-				else:
-					test_prep(p)
-				ret = (
-					ret
-					and test_expected_result(p, test, attempts == 0,
-					                         last_attempt_cb,
-					                         test.get('attempts', 3))
-				)
+			for i, test in enumerate(tests):
+				with suite.test(test.get('name', 'test_{0}'.format(i)),
+				                attempts) as ptest:
+					try:
+						test_prep = test['prep_cb']
+					except KeyError:
+						pass
+					else:
+						test_prep(p)
+					test_result = test_expected_result(p, test, attempts == 0,
+					                                   last_attempt_cb,
+					                                   test.get('attempts', 3))
+					if not test_result:
+						ptest.fail('Result does not match expected')
+				ret = ret and test_result
 
 			if ret:
 				return ret

tests/shlib/common.sh

@@ -13,7 +13,7 @@ if test -z "$FAILED" ; then
 	FAIL_SUMMARY=""
 
 	TMP_ROOT="$ROOT/tests/tmp"
-	FAILURES_FILE="$ROOT/tests/failures"
+	export FAILURES_FILE="$ROOT/tests/failures"
 fi
 
 ANSI_CLEAR="\033[0K"

tests/test_in_vterm/test_tmux.py

@@ -17,6 +17,7 @@ from powerline import get_fallback_logger
 
 from tests.modules.lib.terminal import (ExpectProcess, MutableDimensions,
                                         do_terminal_tests, get_env)
+from tests.modules.lib import PowerlineTestSuite
 
 
 TEST_ROOT = os.path.abspath(os.environ['TEST_ROOT'])
@@ -229,16 +230,18 @@ def main(attempts=3):
 		'new-window', 'bash --norc --noprofile -i', ';',
 	]
 
-	return do_terminal_tests(
-		tests=tests,
-		cmd=tmux_exe,
-		dim=dim,
-		args=args,
-		env=env,
-		cwd=TEST_ROOT,
-		fin_cb=tmux_fin_cb,
-		last_attempt_cb=print_tmux_logs,
-	)
+	with PowerlineTestSuite('tmux') as suite:
+		return do_terminal_tests(
+			tests=tests,
+			cmd=tmux_exe,
+			dim=dim,
+			args=args,
+			env=env,
+			cwd=TEST_ROOT,
+			fin_cb=tmux_fin_cb,
+			last_attempt_cb=print_tmux_logs,
+			suite=suite,
+		)
 
 
 if __name__ == '__main__':

tests/test_in_vterm/test_vim.py

@@ -14,6 +14,7 @@ from powerline.lib.dict import updated
 
 from tests.modules.lib.terminal import (ExpectProcess, MutableDimensions,
                                         do_terminal_tests, get_env)
+from tests.modules.lib import PowerlineTestSuite
 
 
 TEST_ROOT = os.path.abspath(os.environ['TEST_ROOT'])
@@ -53,14 +54,16 @@ def main(attempts=3):
 	tests = (
 	)
 
-	return do_terminal_tests(
-		tests=tests,
-		cmd=vim_exe,
-		dim=dim,
-		args=args,
-		env=env,
-		cwd=TEST_ROOT,
-	)
+	with PowerlineTestSuite('vim') as suite:
+		return do_terminal_tests(
+			tests=tests,
+			cmd=vim_exe,
+			dim=dim,
+			args=args,
+			env=env,
+			cwd=TEST_ROOT,
+			suite=suite,
+		)
 
 
 if __name__ == '__main__':