Refactor output of all vm-tests

etc/init.d/icinga2.cmake

@@ -138,6 +138,7 @@ status() {
 		echo "Running"
 	else
 		echo "Not running"
+		exit 3
 	fi
 }
 

test/jenkins/apache_state.test

@@ -1,3 +1,10 @@
 #!/bin/sh
 
-sudo service httpd status
+sudo service httpd status &> /dev/null
+if [ $? -gt 0 ]; then
+    echo '[FAIL] httpd is not running'
+    exit 1
+else
+    echo '[OK] httpd is running'
+    exit 0
+fi

test/jenkins/checkresult.test

@@ -31,6 +31,16 @@ output=%(output)s
 """
 
 
+def success(msg):
+    print '[OK] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+
+
+def fail(msg):
+    print '[FAIL] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+
+
 def main():
     run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
 
@@ -43,7 +53,7 @@ def main():
             "where c.display_name = 'PassiveService1'"
     state_time = float(next(iter(run_query(query)), {}).get('last_check', '0'))
     if state_time == 0:
-        print '"PassiveService1" seems not to have been checked yet'
+        fail('"PassiveService1" seems not to have been checked yet')
         return 1
 
     if (state_time + CHECK_INTERVAL) - time.time() < 30:
@@ -76,10 +86,10 @@ def main():
             "where c.display_name = 'PassiveService1'"
     output = next(iter(run_query(query)), {}).get('output', '')
     if output != 'Passing in CheckResult header files works!':
-        print 'Checkresult header files seem not to be processed properly'
+        fail('Checkresult header files seem not to be processed properly')
         return 1
 
-    print 'Checkresult header files are processed properly'
+    success('Checkresult header files are processed properly')
     return 0
 
 

test/jenkins/eventhandler.test

@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import os
 import sys
 import time
+import socket
 
 import utils
 
 
+LIVESTATUS_PATH = '/var/run/icinga2/cmd/livestatus'
 LS_HOST_COLUMNS = [
     'name',
     'name',
@@ -108,14 +110,22 @@ def convert_output(value):
 
 def success(msg):
     print '[OK] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
 
 
 def fail(msg):
     print '[FAIL] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
 
 
 def info(msg):
     print '[INFO] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+
+
+def error(msg):
+    print '[ERROR] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
 
 
 def main():
@@ -206,5 +216,10 @@ def main():
 
 
 if __name__ == '__main__':
-    with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
+    try:
+        with utils.LiveStatusSocket(LIVESTATUS_PATH) as LIVESTATUS:
             sys.exit(main())
+    except (OSError, IOError, socket.error), e:
+        error('Could not connect to Livestatus socket: {0} ({1})'
+              ''.format(LIVESTATUS_PATH, unicode(e)))
+

test/jenkins/external_commandpipe.test

@@ -2,9 +2,9 @@
 
 if [ -e "/var/run/icinga2/cmd/icinga2.cmd" ];
 then
-    echo "Icinga2 commandpipe found"
+    echo "[OK] Icinga2 commandpipe found"
     exit 0
 else
-    echo "Icinga2 commandpipe not found"
+    echo "[FAIL] Icinga2 commandpipe not found"
     exit 1
 fi

test/jenkins/external_commands.test

@@ -6,6 +6,7 @@ import os
 import sys
 import time
 import random
+import socket
 import subprocess
 
 try:
@@ -20,6 +21,7 @@ import utils
 ICINGA_UID = 'icinga'
 USERNAME = 'Icinga 2 Admin'
 MAX_CHECK_ATTEMPTS = 3
+LIVESTATUS_PATH = '/var/run/icinga2/cmd/livestatus'
 
 
 def send_command(command, quiet=False):
@@ -61,11 +63,19 @@ def restart_icinga():
 
 def success(msg):
     print '[OK] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
     return False
 
 
 def fail(msg):
     print '[FAIL] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+    return True
+
+
+def error(msg):
+    print '[ERROR] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
     return True
 
 
@@ -548,7 +558,10 @@ def test_disabling_scheduling_host_checks(hostname):
                                      ' INNER JOIN icinga_hosts AS h'
                                      ' ON h.host_object_id = c.host_object_id'
                                      ' WHERE h.alias = "{0}"'
-                                     ''.format(hostname))['e_time'])
+                                     ''.format(hostname)).get('e_time', -1))
+    if execution_time == -1:
+        return fail('Unable to fetch the maximum execution time of'
+                    ' host "{0}" from the IDO'.format(hostname))
 
     send_command('DISABLE_HOST_CHECK;{0}'.format(hostname))
     scheduled_check = time.time() + 4
@@ -602,7 +615,10 @@ def test_disabling_scheduling_service_checks(hostname, servicename):
                                      ' INNER JOIN icinga_hosts as h'
                                      ' ON h.host_object_id = s.host_object_id'
                                      ' WHERE h.alias = "{0}" AND s.display_name = "{1}"'
-                                     ''.format(hostname, servicename))['e_time'])
+                                     ''.format(hostname, servicename)).get('e_time', -1))
+    if execution_time == -1:
+        return fail('Unable to fetch the maximum execution time of service "{0}"'
+                    ' on host "{1}" from the IDO'.format(hostname, servicename))
 
     send_command('DISABLE_SVC_CHECK;{0};{1}'.format(hostname, servicename))
     scheduled_check = time.time() + 4
@@ -1193,6 +1209,10 @@ def main():
 
 
 if __name__ == '__main__':
-    with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
+    try:
+        with utils.LiveStatusSocket(LIVESTATUS_PATH) as LIVESTATUS:
             sys.exit(main())
+    except (OSError, IOError, socket.error), e:
+        error('Could not connect to Livestatus socket: {0} ({1})'
+              ''.format(LIVESTATUS_PATH, unicode(e)))
 

test/jenkins/files/ido_tests.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import sys
 from datetime import datetime, timedelta
 
 CHECK_INTERVAL = 10 # minutes; The actual interval are 5 minutes but as other
@@ -78,20 +79,39 @@ EXAMPLE_CONFIG = {
 }
 
 
+def success(msg):
+    print '[OK] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+    return True
+
+
+def fail(msg):
+    print '[FAIL] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+    return False
+
+
+def info(msg):
+    print '[INFO] {0}'.format(msg).encode('utf-8')
+    sys.stdout.flush()
+
+
 def validate_tables(tables):
     """
     Return whether all tables of the IDO database scheme exist in
     the given table listing
 
     """
-    missing = [n for n in TABLES if TABLE_PREFIX + n not in tables]
+    info('Checking database scheme... (tables)')
-    if missing:
+    failures = False
-        print 'Some tables are missing in the IDO'
+    for table in (TABLE_PREFIX + n for n in TABLES):
-        print 'Missing tables: ' + ', '.join(missing)
+        if table in tables:
-        return False
+            success('Found table "{0}" in database'.format(table))
+        else:
+            fail('Could not find table "{0}" in database'.format(table))
+            failures = True
 
-    print 'All tables were found in the IDO'
+    return not failures
-    return True
 
 
 def verify_host_config(config_data):
@@ -99,12 +119,16 @@ def verify_host_config(config_data):
     Return whether the example hosts exist in the given "hosts" table
 
     """
-    if len([1 for e in config_data if e['alias'] in EXAMPLE_CONFIG]) == 2:
+    info('Checking example host configuration...')
-        print 'All example hosts are stored in the IDO'
+    failures = False
-        return True
+    for hostname in EXAMPLE_CONFIG:
+        if not any(1 for e in config_data if e['alias'] == hostname):
+            fail('Could not find host "{0}"'.format(hostname))
+            failures = True
+        else:
+            success('Found host "{0}"'.format(hostname))
 
-    print 'Some example hosts are missing in the IDO'
+    return not failures
-    return False
 
 
 def verify_service_config(config_data):
@@ -112,17 +136,21 @@ def verify_service_config(config_data):
     Return whether the example services exist in the given "services" table
 
     """
+    info('Checking example service configuration...')
+    failures = False
     for hostname, servicename in ((h, s) for h, ss in EXAMPLE_CONFIG.iteritems()
                                          for s in ss):
-        # Not very efficient, but suitable for just two hosts...
         if not any(1 for c in config_data
                      if c['alias'] == hostname and
                         c['display_name'] == servicename):
-            print 'The config stored in the IDO is missing some services'
+            fail('Could not find service "{0}" on host "{1}"'
-            return False
+                 ''.format(servicename, hostname))
+            failures = True
+        else:
+            success('Found service "{0}" on host "{1}"'
+                    ''.format(servicename, hostname))
 
-    print 'The service config stored in the IDO is correct'
+    return not failures
-    return True
 
 
 def check_last_host_status_update(check_info):
@@ -130,22 +158,30 @@ def check_last_host_status_update(check_info):
     Return whether the example hosts are checked as scheduled
 
     """
-    for info in check_info:
+    info('Checking last host status updates...')
-        if info['alias'] == 'localhost':
+    failures = False
-            last_check = datetime.fromtimestamp(float(info['last_check']))
+    for host_info in check_info:
+        if host_info['alias'] == 'localhost':
+            last_check = datetime.fromtimestamp(float(host_info['last_check']))
             if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
                                                        seconds=10):
-                print 'The last status update of host "localhost"' \
+                fail('The last status update of host "{0}" was more than {1} '
-                      ' was more than {0} minutes ago'.format(CHECK_INTERVAL)
+                     'minutes ago'.format(host_info['alias'], CHECK_INTERVAL))
-                return False
+                failures = True
-        elif info['alias'] == 'nsca-ng':
+            else:
-            if float(info['last_check']) > 0:
+                success('Host "{0}" is being updated'.format(host_info['alias']))
-                print 'The host "nsca-ng" was checked even though' \
+        elif host_info['alias'] == 'nsca-ng':
-                      ' it should not be actively checked'
+            if float(host_info['last_check']) > 0:
-                return False
+                fail('The host "{0}" was checked even though it has'
+                     ' no check service'.format(host_info['alias']))
+                failures = True
+            else:
+                success('Host "{0}" is not being checked because there '
+                        'is no check service'.format(host_info['alias']))
+        else:
+            info('Skipping host "{0}"'.format(host_info['alias']))
 
-    print 'The updates of both example hosts are processed as configured'
+    return not failures
-    return True
 
 
 def check_last_service_status_update(check_info):
@@ -153,19 +189,26 @@ def check_last_service_status_update(check_info):
     Return whether the example services are checked as scheduled
 
     """
-    for info in check_info:
+    info('Checking last service status updates...')
-        if info['display_name'] in EXAMPLE_CONFIG.get(info['alias'], []):
+    failures = False
-            last_check = datetime.fromtimestamp(float(info['last_check']))
+    for svc_info in check_info:
+        if svc_info['display_name'] in EXAMPLE_CONFIG.get(svc_info['alias'], []):
+            last_check = datetime.fromtimestamp(float(svc_info['last_check']))
             if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
                                                        seconds=10):
-                print 'The last status update of service "{0}" of' \
+                fail('The last status update of service "{0}" on'
-                      ' host "{1}" was more than {2} minutes ago' \
+                     ' host "{1}" was more than {2} minutes ago'
-                      ''.format(info['display_name'], info['alias'],
+                     ''.format(svc_info['display_name'], svc_info['alias'],
-                                CHECK_INTERVAL)
+                               CHECK_INTERVAL))
-                return False
+                failures = True
+            else:
+                success('Service "{0}" on host "{1}" is being updated'
+                        ''.format(svc_info['display_name'], svc_info['alias']))
+        else:
+            info('Skipping service "{0}" on host "{1}"'
+                 ''.format(svc_info['display_name'], svc_info['alias']))
 
-    print 'The updates of all example services are processed as configured'
+    return not failures
-    return True
 
 
 def check_logentries(logentry_info):
@@ -174,17 +217,15 @@ def check_logentries(logentry_info):
     and refers to its very last hard status change
 
     """
+    info('Checking status log for host "localhost"...')
     if logentry_info and logentry_info[0]['alias'] == 'localhost':
         entry_time = datetime.fromtimestamp(float(logentry_info[0]['entry_time']))
         state_time = datetime.fromtimestamp(float(logentry_info[0]['state_time']))
         if entry_time - state_time > timedelta(seconds=10):
-            print 'The last hard state of host "localhost"' \
+            return fail('The last hard state of host "localhost"'
-                  ' seems not to have been logged'
+                        ' seems not to have been logged')
-            return False
     else:
-        print 'No logs found in the IDO for host "localhost"'
+        return fail('No logs found in the IDO for host "localhost"')
-        return False
 
-    print 'The last hard state of host "localhost" was properly logged'
+    return success('The last hard state of host "localhost" was properly logged')
-    return True
 

test/jenkins/icinga2_state.test

@@ -1,3 +1,10 @@
 #!/bin/sh
 
-sudo service icinga2 status
+sudo service icinga2 status &> /dev/null
+if [ $? -gt 0 ]; then
+    echo '[FAIL] icinga2 is not running'
+    exit 1
+else
+    echo '[OK] icinga2 is running'
+    exit 0
+fi

test/jenkins/ido_mysql.test

@@ -8,15 +8,17 @@ import ido_tests
 
 
 def main():
+    failures = False
+
     run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
 
     if not ido_tests.validate_tables([d['Tables_in_icinga']
                                       for d in run_query('show tables')]):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     host_info = run_query('select * from icinga_hosts')
     if not ido_tests.verify_host_config(host_info):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     service_info = run_query(
         'select c2.alias, c1.* from icinga_services as c1 '
@@ -24,7 +26,7 @@ def main():
         ' on c1.host_object_id = c2.host_object_id'
         )
     if not ido_tests.verify_service_config(service_info):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     hostchecks_data = run_query(
         'select c.alias, unix_timestamp(s.last_check) as last_check'
@@ -33,7 +35,7 @@ def main():
         ' on s.host_object_id = c.host_object_id'
         )
     if not ido_tests.check_last_host_status_update(hostchecks_data):
-        return 1
+        failures = True
 
     servicechecks_data = run_query(
         'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
@@ -44,7 +46,7 @@ def main():
         ' on c1.host_object_id = c2.host_object_id'
         )
     if not ido_tests.check_last_service_status_update(servicechecks_data):
-        return 1
+        failures = True
 
     logentry_info = run_query(
         'select hosts.alias,'
@@ -57,9 +59,9 @@ def main():
         ' on hist.object_id = hosts.host_object_id and hist.state_type = 1'
         )
     if not ido_tests.check_logentries(logentry_info):
-        return 1
+        failures = True
 
-    return 0
+    return 1 if failures else 0
 
 
 if __name__ == '__main__':

test/jenkins/ido_pgsql.test

@@ -8,15 +8,17 @@ import ido_tests
 
 
 def main():
+    failures = False
+
     run_query = lambda q: utils.run_pgsql_query(q, b'/usr/bin/psql')
 
     if not ido_tests.validate_tables([d['Name'] for d in run_query('\\dt')
                                                 if d['Type'] == 'table']):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     host_info = run_query('select * from icinga_hosts')
     if not ido_tests.verify_host_config(host_info):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     service_info = run_query(
         'select c2.alias, c1.* from icinga_services as c1 '
@@ -24,7 +26,7 @@ def main():
         ' on c1.host_object_id = c2.host_object_id'
         )
     if not ido_tests.verify_service_config(service_info):
-        return 1
+        return 1 # Bail out as we cannot proceed without any data
 
     hostchecks_data = run_query(
         'select c.alias, unix_timestamp(s.last_check) as last_check'
@@ -33,7 +35,7 @@ def main():
         ' on s.host_object_id = c.host_object_id'
         )
     if not ido_tests.check_last_host_status_update(hostchecks_data):
-        return 1
+        failures = True
 
     servicechecks_data = run_query(
         'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
@@ -44,7 +46,7 @@ def main():
         ' on c1.host_object_id = c2.host_object_id'
         )
     if not ido_tests.check_last_service_status_update(servicechecks_data):
-        return 1
+        failures = True
 
     logentry_info = run_query(
         'select hosts.alias,'
@@ -59,9 +61,9 @@ def main():
         'group by hosts.alias'
         )
     if not ido_tests.check_logentries(logentry_info):
-        return 1
+        failures = True
 
-    return 0
+    return 1 if failures else 0
 
 
 if __name__ == '__main__':

test/jenkins/livestatus_socket.test

@@ -8,10 +8,10 @@ then
 
     if [ ! -e /var/run/icinga2/cmd/livestatus ];
     then
-        echo "Icinga2 Livestatus socket not found"
+        echo "[FAIL] Icinga2 Livestatus socket not found"
         exit 1
     fi
 fi
 
-echo "Icinga2 Livestatus socket found"
+echo "[OK] Icinga2 Livestatus socket found"
 exit 0

test/jenkins/logfile.test

@@ -2,9 +2,9 @@
 
 if sudo test -f /var/log/icinga2/icinga2.log;
 then
-    echo "Icinga2 log file found"
+    echo "[OK] Icinga2 log file found"
     exit 0
 else
-    echo "Icinga2 log file not found"
+    echo "[FAIL] Icinga2 log file not found"
     exit 1
 fi

test/jenkins/mysql_state.test

@@ -1,3 +1,10 @@
 #!/bin/sh
 
-sudo service mysqld status
+sudo service mysqld status &> /dev/null
+if [ $? -gt 0 ]; then
+    echo '[FAIL] mysqld is not running'
+    exit 1
+else
+    echo '[OK] mysqld is running'
+    exit 0
+fi

test/jenkins/nsca-ng.test

@@ -6,24 +6,24 @@ sleep 3
 pid_after_restart=`ps -U icinga | grep icinga2 | awk '{print $1}'`
 
 if [ $pid_after_restart -eq $pid_before_restart ]; then
-    echo "Failed to send 'RESTART_PROCESS' to icinga2"
+    echo "[FAIL] Failed to send 'RESTART_PROCESS' to icinga2"
     exit 1
 else
-    echo "Successfully sent 'RESTART_PROCESS' to icinga2"
+    echo "[OK] Successfully sent 'RESTART_PROCESS' to icinga2"
 fi
 
 printf "localhost\t0\tA passive result returning OK\n" | sudo send_nsca
 if [ $? -gt 0 ]; then
-    echo "Failed to send passive check result for host 'localhost'"
+    echo "[FAIL] Failed to send passive check result for host 'localhost'"
     exit 1
 else
-    echo "Successfully sent a passive check result for host 'localhost'"
+    echo "[OK] Successfully sent a passive check result for host 'localhost'"
 fi
 
 printf "localhost\tdisk\t2\tA passive result not returning OK\n" | sudo send_nsca
 if [ $? -gt 0 ]; then
-    echo "Failed to send passive check result for service 'disk' on host 'localhost'"
+    echo "[FAIL] Failed to send passive check result for service 'disk' on host 'localhost'"
     exit 1
 else
-    echo "Successfully sent a passive check result for service 'disk' on host 'localhost'"
+    echo "[OK] Successfully sent a passive check result for service 'disk' on host 'localhost'"
 fi

test/jenkins/pgsql_state.test

@@ -1,3 +1,10 @@
 #!/bin/sh
 
-sudo service postgresql status
+sudo service postgresql status &> /dev/null
+if [ $? -gt 0 ]; then
+    echo '[FAIL] postgresql is not running'
+    exit 1
+else
+    echo '[OK] postgresql is running'
+    exit 0
+fi

test/jenkins/pidfile.test

@@ -2,9 +2,9 @@
 
 if [ -f /var/run/icinga2/icinga2.pid ];
 then
-    echo "Icinga2 pidfile found"
+    echo "[OK] Icinga2 pidfile found"
     exit 0
 else
-    echo "Icinga2 pidfile not found"
+    echo "[FAIL] Icinga2 pidfile not found"
     exit 1
 fi

test/jenkins/run_tests.py

@@ -18,6 +18,49 @@ except ImportError:
     DEVNULL = open(os.devnull, 'w')
 
 
+class Logger(object):
+    INFO = 1
+    ERROR = 2
+    FAIL = 3
+    OK = 4
+
+    @staticmethod
+    def write(text, stderr=False):
+        if stderr:
+            sys.stderr.write(text)
+            sys.stderr.flush()
+        else:
+            sys.stdout.write(text)
+            sys.stdout.flush()
+
+    @classmethod
+    def log(cls, severity, text):
+        if severity == cls.INFO:
+            cls.write('\033[1;94m[INFO]\033[1;0m {0}'.format(text))
+        elif severity == cls.ERROR:
+            cls.write('\033[1;33m[ERROR]\033[1;0m {0}'.format(text), True)
+        elif severity == cls.FAIL:
+            cls.write('\033[1;31m[FAIL] {0}\033[1;0m'.format(text))
+        elif severity == cls.OK:
+            cls.write('\033[1;32m[OK]\033[1;0m {0}'.format(text))
+
+    @classmethod
+    def info(cls, text):
+        cls.log(cls.INFO, text)
+
+    @classmethod
+    def error(cls, text):
+        cls.log(cls.ERROR, text)
+
+    @classmethod
+    def fail(cls, text):
+        cls.log(cls.FAIL, text)
+
+    @classmethod
+    def ok(cls, text):
+        cls.log(cls.OK, text)
+
+
 class TestSuite(object):
     def __init__(self, configpath):
         self._tests = []
@@ -64,17 +107,26 @@ class TestSuite(object):
     def run(self):
         for path in self._tests:
             test_name = os.path.basename(path)
-            self._apply_setup_routines(test_name, 'setup')
+            Logger.info('Copying test "{0}" to remote machine\n'.format(test_name))
             self._copy_test(path)
-            self._results[test_name] = self._run_test(path)
+            self._apply_setup_routines(test_name, 'setup')
+            Logger.info('Running test "{0}"...\n'.format(test_name))
+            result = self._run_test(path)
+            Logger.info('Test "{0}" has finished (Total tests: {1}, Failures: {2})\n'
+                        ''.format(test_name, result['total'], result['failures']))
             self._apply_setup_routines(test_name, 'teardown')
+            Logger.info('Removing test "{0}" from remote machine\n'.format(test_name))
             self._remove_test(test_name)
+            self._results[test_name] = result
+            Logger.write('\n')
 
     def _apply_setup_routines(self, test_name, context):
         instructions = next((t[1].get(context)
                              for t in self._config.get('setups', {}).iteritems()
                              if re.match(t[0], test_name)), None)
         if instructions is not None:
+            Logger.info('Applying {0} routines for test "{1}" .. '
+                        ''.format(context, test_name))
             for instruction in instructions.get('copy', []):
                 source, _, destination = instruction.partition('>>')
                 self._copy_file(source.strip(), destination.strip())
@@ -82,26 +134,27 @@ class TestSuite(object):
                 self._remove_file(filepath)
             for command in instructions.get('exec', []):
                 self._exec_command(command)
+            Logger.write('Done\n')
 
     def _remove_file(self, path):
         command = self._config['commands']['clean'].format(path)
         rc = subprocess.call(command, stdout=DEVNULL, shell=True)
         if rc != 0:
-            print 'WARNING: Cannot remove file "{0}" ({1})'.format(path, rc)
+            Logger.error('Cannot remove file "{0}" ({1})\n'.format(path, rc))
 
     def _exec_command(self, command):
         command = self._config['commands']['exec'].format(command)
         rc = subprocess.call(command, stdout=DEVNULL, shell=True)
         if rc != 0:
-            print 'WARNING: Command "{0}" exited with exit code "{1}"' \
+            Logger.error('Command "{0}" exited with exit code "{1}"' \
-                  ''.format(command, rc)
+                         ''.format(command, rc))
 
     def _copy_file(self, source, destination):
         command = self._config['commands']['copy'].format(source, destination)
         rc = subprocess.call(command, stdout=DEVNULL, shell=True)
         if rc != 0:
-            print 'WARNING: Cannot copy file "{0}" to "{1}" ({2})' \
+            Logger.error('Cannot copy file "{0}" to "{1}" ({2})' \
-                  ''.format(source, destination, rc)
+                         ''.format(source, destination, rc))
 
     def _copy_test(self, path):
         self._copy_file(path, os.path.join(self._config['settings']['test_root'],
@@ -117,22 +170,44 @@ class TestSuite(object):
                               os.path.basename(path))
         p = subprocess.Popen(command.format(target), stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE, shell=True)
-        out, err = p.communicate()
+        output, test_count, failed_tests = self._watch_output(p.stdout)
-
         return {
-            'stdout': out.decode('utf-8'),
+            'total': test_count,
-            'stderr': err.decode('utf-8'),
+            'failures': failed_tests,
-            'returncode': p.returncode
+            'stdout': output,
+            'stderr': p.stderr.read().decode('utf-8'),
+            'returncode': p.wait()
             }
 
+    def _watch_output(self, pipe):
+        output, total, failures = '', 0, 0
+        while True:
+            line = pipe.readline().decode('utf-8')
+            if not line:
+                break
+
+            if line.startswith('[ERROR] '):
+                Logger.error(line[8:])
+            elif line.startswith('[FAIL] '):
+                Logger.fail(line[7:])
+                failures += 1
+                total += 1
+            elif line.startswith('[OK] '):
+                Logger.ok(line[5:])
+                total += 1
+            else:
+                Logger.info(line.replace('[INFO] ', ''))
+
+            output += line
+        return (output, total, failures)
+
 
 def parse_commandline():
-    parser = OptionParser(version='0.1')
+    parser = OptionParser(version='0.2')
     parser.add_option('-C', '--config', default="run_tests.conf",
                       help='The path to the config file to use [%default]')
-    parser.add_option('-O', '--output',
+    parser.add_option('-R', '--results',
-                      help='The file which to save the test results. '
+                      help='The file where to store the test results')
-                           '(By default this goes to stdout)')
     return parser.parse_args()
 
 
@@ -145,12 +220,9 @@ def main():
 
     suite.run()
 
-    report = suite.get_report()
+    if options.results is not None:
-    if options.output is None:
+        with open(options.results, 'w') as f:
-        print report.encode('utf-8')
+            f.write(suite.get_report().encode('utf-8'))
-    else:
-        with open(options.output, 'w') as f:
-            f.write(report.encode('utf-8'))
 
     return 0
 

test/jenkins/statusdata.test

@@ -20,18 +20,18 @@ then
 
     if [ $n -eq 3 ];
     then
-        echo "Icinga2 status.dat not found"
+        echo "[FAIL] Icinga2 status.dat not found"
         exit 1
     fi
 fi
 
-echo "Icinga2 status.dat found"
+echo "[OK] Icinga2 status.dat found"
 
 if [ -f /var/cache/icinga2/objects.cache ];
 then
-    echo "Icinga2 objects.cache found"
+    echo "[OK] Icinga2 objects.cache found"
 else
-    echo "Icinga2 objects.cache not found"
+    echo "[FAIL] Icinga2 objects.cache not found"
     exit 1
 fi
 
@@ -44,8 +44,8 @@ new_status_time=$(stat --format="%Y" /var/cache/icinga2/status.dat)
 
 if [ $new_status_time -eq $status_time ];
 then
-    echo "Icinga2 status.dat is not being updated"
+    echo "[FAIL] Icinga2 status.dat is not being updated"
     exit 1
 else
-    echo "Icinga2 status.dat is being updated"
+    echo "[OK] Icinga2 status.dat is being updated"
 fi