Use tests/common.sh for reporting test failures

tests/common.sh

@@ -0,0 +1,33 @@
+. tests/bot-ci/scripts/common/main.sh
+set +x
+
+: ${PYTHON:=python}
+
+FAILED=0
+
+FAIL_SUMMARY=""
+
+enter_suite() {
+	local suite_name="$1"
+	export POWERLINE_CURRENT_SUITE="${POWERLINE_CURRENT_SUITE}/$suite_name"
+}
+
+exit_suite() {
+	if test $FAILED -ne 0 ; then
+		echo "Suite ${POWERLINE_CURRENT_SUITE} failed, summary:"
+		echo "${FAIL_SUMMARY}"
+	fi
+	export POWERLINE_CURRENT_SUITE="${POWERLINE_CURRENT_SUITE%/*}"
+	exit $FAILED
+}
+
+fail() {
+	local test_name="$1"
+	local fail_char="$2"
+	local message="$3"
+	local full_msg="$fail_char $POWERLINE_CURRENT_SUITE|$test_name :: $message"
+	FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+	echo "Failed: $full_msg"
+	echo "$full_msg" >> tests/failures
+	FAILED=1
+}

tests/run_daemon_tests.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
-FAILED=0
+. tests/common.sh
+
+enter_suite daemon
+
 export ADDRESS="powerline-ipc-test-$$"
 echo "Powerline address: $ADDRESS"
 if $PYTHON scripts/powerline-daemon -s$ADDRESS ; then
@@ -8,16 +11,14 @@ if $PYTHON scripts/powerline-daemon -s$ADDRESS ; then
 		$PYTHON client/powerline.py --socket $ADDRESS -p/dev/null shell left | \
 		grep 'file not found'
 	) ; then
-		echo "-p/dev/null argument ignored or not treated properly"
+		fail "devnull" F "-p/dev/null argument ignored or not treated properly"
-		FAILED=1
 	fi
 	if ( \
 		$PYTHON client/powerline.py --socket $ADDRESS \
 			-p$PWD/powerline/config_files shell left | \
 		grep 'file not found'
 	) ; then
-		echo "-p/dev/null argument remembered while it should not"
+		fail "nodevnull" F "-p/dev/null argument remembered while it should not"
-		FAILED=1
 	fi
 	if ! ( \
 		cd tests && \
@@ -25,17 +26,15 @@ if $PYTHON scripts/powerline-daemon -s$ADDRESS ; then
 			-p$PWD/../powerline/config_files shell left | \
 		grep 'tests'
 	) ; then
-		echo "Output lacks string “tests”"
+		fail "segment" F "Output lacks string “tests”"
-		FAILED=1
 	fi
 else
-	echo "Daemon exited with status $?"
+	fail "exitcode" E "Daemon exited with status $?"
-	FAILED=1
 fi
 if $PYTHON scripts/powerline-daemon -s$ADDRESS -k ; then
 	:
 else
-	echo "powerline-daemon -k failed with exit code $?"
+	fail "-k" F "powerline-daemon -k failed with exit code $?"
-	FAILED=1
 fi
-exit $FAILED
+
+exit_suite

tests/run_lint_tests.sh

@@ -1,7 +1,10 @@
 #!/bin/sh
-FAILED=0
+. tests/common.sh
+
+enter_suite lint
+
 if ! ${PYTHON} scripts/powerline-lint -p powerline/config_files ; then
-	echo "Failed powerline-lint"
+	fail "test" F "Running powerline-lint failed"
-	FAILED=1
 fi
-exit $FAILED
+
+exit_suite

tests/run_python_tests.sh

@@ -1,9 +1,13 @@
 #!/bin/sh
-FAILED=0
+. tests/common.sh
+
+enter_suite python
+
 for file in tests/test_*.py ; do
+	test_name="${file##*/test_}"
 	if ! ${PYTHON} $file --verbose --catch ; then
-		echo "Failed test(s) from $file"
+		fail "${test_name%.py}" F "Failed test(s) from $file"
-		FAILED=1
 	fi
 done
-exit $FAILED
+
+exit_suite

tests/run_vim_tests.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
-. tests/bot-ci/scripts/common/main.sh
+. tests/common.sh
-FAILED=0
+
+enter_suite vim
 
 if test -z "$VIM" ; then
 	if test -n "$USE_UCS2_PYTHON" ; then
@@ -42,28 +43,29 @@ export POWERLINE_THEME_OVERRIDES='default.segments.left=[]'
 test_script() {
 	local vim="$1"
 	local script="$2"
+	local test_name_prefix="$3"
 	echo "Running script $script with $vim"
 	if ! test -e "$vim" ; then
 		return 0
 	fi
 	if ! "$vim" -u NONE -S $script || test -f message.fail ; then
-		echo "Failed script $script run with $VIM" >&2
+		local test_name="$test_name_prefix-${script##*/}"
+		fail "${test_name%.vim}" F "Failed script $script run with $VIM"
 		cat message.fail >&2
 		rm message.fail
-		FAILED=1
 	fi
 }
 
 for script in tests/test_*.vim ; do
 	if test "${script%.old.vim}" = "${script}" ; then
-		test_script "$NEW_VIM" "$script"
+		test_script "$NEW_VIM" "$script" new
 	fi
 done
 
 if test -e "$OLD_VIM" ; then
 	for script in tests/test_*.old.vim ; do
-		test_script "$OLD_VIM" "$script"
+		test_script "$OLD_VIM" "$script" old
 	done
 fi
 
-exit $FAILED
+exit_suite

tests/run_vterm_tests.sh

@@ -1,8 +1,13 @@
 #!/bin/sh
-set -e
+. tests/common.sh
-FAILED=0
+
-if ! sh tests/test_in_vterm/test_tmux.sh ; then
+enter_suite vterm
-	echo "Failed vterm"
+
-	FAILED=1
+for t in tests/test_in_vterm/test_*.sh ; do
+	test_name="${t##*/test_}"
+	if ! sh "$t" ; then
+		fail "${test_name%.sh}" F "Failed running $t"
 	fi
-exit $FAILED
+done
+
+exit_suite

tests/test.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-. tests/bot-ci/scripts/common/main.sh
+. tests/common.sh
+
+enter_suite root
 
 : ${USER:=`id -un`}
 : ${HOME:=`getent passwd $USER | cut -d: -f6`}
 
 export USER HOME
 
-FAILED=0
-
 if test "$TRAVIS" = true ; then
 	export PATH="$HOME/opt/fish/bin:${PATH}"
 	export PATH="$PWD/tests/bot-ci/deps/rc:$PATH"
@@ -34,9 +34,16 @@ fi
 export PYTHON="${PYTHON:=python}"
 export PYTHONPATH="${PYTHONPATH}${PYTHONPATH:+:}`realpath .`"
 for script in tests/run_*_tests.sh ; do
+	test_name="${script##*/run_}"
 	if ! sh $script ; then
-		echo "Failed $script"
+		fail "${test_name%_tests.sh}" F "Failed $script"
-		FAILED=1
 	fi
 done
-exit $FAILED
+
+if test -e tests/failures ; then
+	echo "Some tests failed. Summary:"
+	cat tests/failures
+	rm tests/failures
+fi
+
+exit_suite

tests/test_in_vterm/test_tmux.sh

@@ -1,8 +1,7 @@
 #!/bin/sh
-. tests/bot-ci/scripts/common/main.sh
+. tests/common.sh
-set +x
 
-FAILED=0
+enter_suite tmux
 
 rm -rf tests/vterm_tmux
 mkdir tests/vterm_tmux
@@ -17,8 +16,6 @@ ln -s "$PWD/scripts/powerline-config" tests/vterm_tmux/path
 
 cp -r tests/terminfo tests/vterm_tmux
 
-FAIL_SUMMARY=""
-
 test_tmux() {
 	if test "$PYTHON_IMPLEMENTATION" = PyPy; then
 		# FIXME PyPy3 segfaults for some reason, PyPy does it as well, but 
@@ -31,9 +28,8 @@ test_tmux() {
 	ln -sf "$(which "${POWERLINE_TMUX_EXE}")" tests/vterm_tmux/path
 	f=tests/test_in_vterm/test_tmux.py
 	if ! "${PYTHON}" $f ; then
-		echo "Failed vterm test $f"
+		local test_name="$("$POWERLINE_TMUX_EXE" -V 2>&1 | cut -d' ' -f2)"
-		FAILED=1
+		fail "$test_name" F "Failed vterm test $f"
-		FAIL_SUMMARY="$FAIL_SUMMARY${NL}F $POWERLINE_TMUX_EXE $f"
 	fi
 }
 
@@ -48,8 +44,9 @@ else
 fi
 
 if test $FAILED -eq 0 ; then
-	echo "$FAIL_SUMMARY"
 	rm -rf tests/vterm_tmux
+else
+	echo "$FAIL_SUMMARY"
 fi
 
-exit $FAILED
+exit_suite

tests/test_shells/test.sh

@@ -1,10 +1,8 @@
 #!/bin/sh
-. tests/bot-ci/scripts/common/main.sh
+. tests/common.sh
-set +x
+
+enter_suite shells
 
-: ${PYTHON:=python}
-FAIL_SUMMARY=""
-FAILED=0
 if test "x$1" = "x--fast" ; then
 	FAST=1
 	shift
@@ -285,9 +283,7 @@ check_test_client() {
 	esac
 	expected_mime_type="${expected_mime_type%/*}"
 	if test "$expected_mime_type" != "$actual_mime_type" ; then
-		echo "Expected $executable to have MIME type $expected_mime_type, but got $actual_mime_type"
+		fail "MIME-$executable" "M" "Expected $executable to have MIME type $expected_mime_type, but got $actual_mime_type"
-		FAILED=1
-		FAIL_SUMMARY="${FAIL_SUMMARY}${NL}M ${executable}"
 	fi
 }
 
@@ -382,8 +378,7 @@ if test -z "${ONLY_SHELL}" || test "x${ONLY_SHELL%sh}" != "x${ONLY_SHELL}" || te
 				fi
 				echo ">>> $(readlink "tests/shell/path/$SH")"
 				if ! run_test $TEST_TYPE $TEST_CLIENT $TEST_COMMAND ; then
-					FAILED=1
+					fail "$SH-$TEST_TYPE-$TEST_CLIENT:test" F "Failed checking $TEST_COMMAND"
-					FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T ${TEST_TYPE} ${TEST_CLIENT} ${TEST_COMMAND}"
 				fi
 			done
 		done
@@ -395,8 +390,7 @@ if test -z "${ONLY_SHELL}" || test "x${ONLY_SHELL%sh}" != "x${ONLY_SHELL}" || te
 				echo "Daemon log:"
 				echo '============================================================'
 				cat tests/shell/daemon_log
-				FAILED=1
+				fail "$SH-$TEST_TYPE-$TEST_CLIENT:log" E "Non-empty daemon log for ${TEST_COMMAND}"
-				FAIL_SUMMARY="${FAIL_SUMMARY}${NL}L ${TEST_TYPE} ${TEST_CLIENT} ${TEST_COMMAND}"
 			fi
 		fi
 	done
@@ -406,9 +400,7 @@ if $PYTHON scripts/powerline-daemon -s$ADDRESS > tests/shell/daemon_log_2 2>&1 ;
 	sleep 1
 	$PYTHON scripts/powerline-daemon -s$ADDRESS -k
 else
-	echo "Daemon exited with status $?"
+	fail "daemon:run" F "Daemon exited with status $?"
-	FAILED=1
-	FAIL_SUMMARY="${FAIL_SUMMARY}${NL}D"
 fi
 
 if ! test -z "$(cat tests/shell/daemon_log_2)" ; then
@@ -416,8 +408,7 @@ if ! test -z "$(cat tests/shell/daemon_log_2)" ; then
 	echo "Daemon log (2nd):"
 	echo '============================================================'
 	cat tests/shell/daemon_log_2
-	FAILED=1
+	fail "daemon:log" E "Daemon run with non-empty log"
-	FAIL_SUMMARY="${FAIL_SUMMARY}${NL}L"
 fi
 
 if ( test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xzsh" ) \
@@ -425,8 +416,7 @@ if ( test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xzsh" ) \
 	&& zsh tests/test_shells/zsh_test_script.zsh 2>/dev/null; then
 	echo "> zpython"
 	if ! run_test zpython zpython zsh -f -i ; then
-		FAILED=1
+		fail "zsh-zpython:test" F "Failed checking zsh -f -i"
-		FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T zpython zsh -f -i"
 	fi
 fi
 
@@ -435,8 +425,7 @@ if  test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xpdb" ; then
 		if test "x${ONLY_TEST_TYPE}" = "x" || test "x${ONLY_TEST_TYPE}" = "xsubclass" ; then
 			echo "> pdb subclass"
 			if ! run_test subclass python $PDB_PYTHON "$PWD/tests/test_shells/pdb-main.py" ; then
-				FAILED=1
+				fail "pdb-subclass:test" F "Failed checking $PDB_PYTHON $PWD/tests/test_shells/pdb-main.py"
-				FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T pdb $PDB_PYTHON $PWD/tests/test_shells/pdb-main.py"
 			fi
 		fi
 		if test "x${ONLY_TEST_TYPE}" = "x" || test "x${ONLY_TEST_TYPE}" = "xmodule" ; then
@@ -446,8 +435,7 @@ if  test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xpdb" ; then
 				MODULE="powerline.bindings.pdb.__main__"
 			fi
 			if ! run_test module python $PDB_PYTHON -m$MODULE "$PWD/tests/test_shells/pdb-script.py" ; then
-				FAILED=1
+				fail "pdb-module:test" F "Failed checking $PDB_PYTHON -m$MODULE $PWD/tests/test_shells/pdb-script"
-				FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T pdb $PDB_PYTHON -m$MODULE $PWD/tests/test_shells/pdb-script"
 			fi
 		fi
 	fi
@@ -460,8 +448,7 @@ if test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xipython" ; then
 		export POWERLINE_THEME_OVERRIDES='in.segments.left=[]'
 		echo "> ipython"
 		if ! run_test ipython ipython ${IPYTHON_PYTHON} -mIPython ; then
-			FAILED=1
+			fail "ipython:test" F "Failed checking ${IPYTHON_PYTHON} -mIPython"
-			FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T ipython"
 		fi
 		unset POWERLINE_THEME_OVERRIDES
 		unset POWERLINE_CONFIG_OVERRIDES
@@ -470,7 +457,6 @@ fi
 
 if test $FAILED -eq 0 ; then
 	rm -r tests/shell
-else
-	echo "${FAIL_SUMMARY}"
 fi
-exit $FAILED
+
+exit_suite