From 9d87ce571b9517abaf02cf58e0a3b1f976f00a07 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Wed, 4 Dec 2013 16:04:28 +0100
Subject: [PATCH 01/17] Add configuration aware test runner

refs #5223
---
 test/jenkins/README         |  43 +++++++++++-
 test/jenkins/run-tests.py   |  34 ----------
 test/jenkins/run_tests.conf |  12 ++++
 test/jenkins/run_tests.py   | 131 ++++++++++++++++++++++++++++++++++++
 test/jenkins/run_tests.sh   |   5 ++
 5 files changed, 190 insertions(+), 35 deletions(-)
 delete mode 100755 test/jenkins/run-tests.py
 create mode 100644 test/jenkins/run_tests.conf
 create mode 100755 test/jenkins/run_tests.py
 create mode 100755 test/jenkins/run_tests.sh

diff --git a/test/jenkins/README b/test/jenkins/README
index 23495e8f2..ff0681810 100644
--- a/test/jenkins/README
+++ b/test/jenkins/README
@@ -1 +1,42 @@
-These scripts are used by build.icinga.org to set up a test VM.
+Set of scripts to set up and test a virtual demo machine
+========================================================
+
+This directory contains a few scripts primarily used by build.icinga.org.
+
+* bootstrap-vm.sh
+  Ensures that all required software is installed and its configuration
+  is applied to the VM. (Usually not of interest for the typical user.)
+
+* run_tests.sh
+  This is a wrapper script intended to be ran manually by a user.
+
+* run_tests.py
+  The actual test-runner. Accepts one option (-C|--config) and expects
+  one or more filenames or -patterns that should be run on the VM.
+
+* run_tests.conf
+  The default configuration file for the test-runner. (Used when running
+  the wrapper script or when no custom configuration file is passed to the
+  test-runner.)
+
+  Format:
+  - commands: This section is mandatory and contains the commands to use.
+  - settings: This section is mandatory and defines settings that are applied to
+              all tests.
+  - setups: This section is optional and contains setup routines that should
+            be ran before (setup) and after (teardown) any matching test is
+            executed. (Note that only one setup can be effective at a time.)
+
+            Example:
+            "^v[1-9]\.test$": {
+                "setup": {
+                    "copy": ["source >> target"], // Files that should be copied.
+                                                  // Note that these files remain
+                                                  // if not removed explicitly
+                    "clean": ["target"], // Files to delete from the system
+                    "exec": ["cmd1", "cmd2"] // Commands to execute on the system
+                },
+                "teardown": {
+                    // The same kind of instructions as above can be added here
+                }
+            }
diff --git a/test/jenkins/run-tests.py b/test/jenkins/run-tests.py
deleted file mode 100755
index e96395553..000000000
--- a/test/jenkins/run-tests.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-import sys
-from xml.dom.minidom import getDOMImplementation
-from subprocess import Popen, PIPE
-
-impl = getDOMImplementation()
-result = impl.createDocument(None, "testsuite", None)
-testsuite = result.documentElement
-
-for fn in sys.argv[1:]:
-  process = Popen(["./" + fn], stdout=PIPE, stderr=PIPE)
-  (stdoutdata, stderrdata) = process.communicate()
-
-  testcase = result.createElement("testcase")
-  testcase.setAttribute("classname", "vm")
-  testcase.setAttribute("name", fn)
-
-  systemout = result.createElement("system-out")
-  systemout.appendChild(result.createTextNode(stdoutdata))
-  testcase.appendChild(systemout)
-
-  systemerr = result.createElement("system-err")
-  systemerr.appendChild(result.createTextNode(stderrdata))
-  testcase.appendChild(systemerr)
-
-  if process.returncode != 0:
-    failure = result.createElement("failure")
-    failure.setAttribute("type", "returncode")
-    failure.appendChild(result.createTextNode("code: " + str(process.returncode)))
-    testcase.appendChild(failure)
-
-  testsuite.appendChild(testcase)
-
-print result.toxml()
diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
new file mode 100644
index 000000000..87a7000ad
--- /dev/null
+++ b/test/jenkins/run_tests.conf
@@ -0,0 +1,12 @@
+{
+    "commands": {
+        "copy": "scp -qF ssh_config {0} default:{1}",
+        "exec": "ssh -F ssh_config default {0}",
+        "clean": "ssh -F ssh_config default rm -f {0}"
+    },
+    "tests": {
+        "destination": "/tmp"
+    },
+    "setups": {
+    }
+}
diff --git a/test/jenkins/run_tests.py b/test/jenkins/run_tests.py
new file mode 100755
index 000000000..12612b514
--- /dev/null
+++ b/test/jenkins/run_tests.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+
+import os
+import re
+import sys
+import json
+import glob
+import subprocess
+from optparse import OptionParser
+from xml.dom.minidom import getDOMImplementation
+
+
+class TestSuite(object):
+    def __init__(self, configpath):
+        self._tests = []
+        self._results = {}
+
+        self.load_config(configpath)
+
+    def add_test(self, filepath):
+        self._tests.append(filepath)
+
+    def load_config(self, filepath):
+        with open(filepath) as f:
+            self._config = json.load(f)
+
+    def get_report(self):
+        dom = getDOMImplementation()
+        document = dom.createDocument(None, 'testsuite', None)
+        xml_root = document.documentElement
+
+        for name, info in self._results.iteritems():
+            testresult = document.createElement('testcase')
+            testresult.setAttribute('classname', 'vm')
+            testresult.setAttribute('name', name)
+
+            systemout = document.createElement('system-out')
+            systemout.appendChild(document.createTextNode(info['stdout']))
+            testresult.appendChild(systemout)
+
+            systemerr = document.createElement('system-err')
+            systemerr.appendChild(document.createTextNode(info['stderr']))
+            testresult.appendChild(systemerr)
+
+            if info['returncode'] != 0:
+                failure = document.createElement('failure')
+                failure.setAttribute('type', 'returncode')
+                failure.appendChild(document.createTextNode(
+                    'code: {0}'.format(info['returncode'])))
+                testresult.appendChild(failure)
+
+            xml_root.appendChild(testresult)
+
+        return document.toxml()
+
+    def run(self):
+        for path in self._tests:
+            test_name = os.path.basename(path)
+            self._apply_setup_routines(test_name, 'setup')
+            self._copy_test(path)
+            self._results[test_name] = self._run_test(path)
+            self._apply_setup_routines(test_name, 'teardown')
+
+    def _apply_setup_routines(self, test_name, context):
+        instructions = next((t[1].get(context)
+                             for t in self._config.get('setups', {}).iteritems()
+                             if re.match(t[0], test_name)), None)
+        if instructions is not None:
+            for instruction in instructions.get('copy', []):
+                source, _, destination = instruction.partition('>>')
+                self._copy_file(source.strip(), destination.strip())
+            for filepath in instructions.get('clean', []):
+                self._remove_file(filepath)
+            for command in instructions.get('exec', []):
+                self._exec_command(command)
+
+    def _remove_file(self, path):
+        command = self._config['commands']['clean'].format(path)
+        subprocess.call(command, shell=True)
+
+    def _exec_command(self, command):
+        command = self._config['commands']['exec'].format(command)
+        subprocess.call(command, shell=True)
+
+    def _copy_file(self, source, destination):
+        command = self._config['commands']['copy'].format(source, destination)
+        subprocess.call(command, shell=True)
+
+    def _copy_test(self, path):
+        self._copy_file(path, os.path.join(self._config['tests']['destination'],
+                                           os.path.basename(path)))
+
+    def _run_test(self, path):
+        command = self._config['commands']['exec']
+        target = os.path.join(self._config['tests']['destination'],
+                              os.path.basename(path))
+        p = subprocess.Popen(command.format(target), stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE, shell=True)
+        out, err = p.communicate()
+
+        return {
+            'stdout': out.decode('utf-8'),
+            'stderr': err.decode('utf-8'),
+            'returncode': p.returncode
+            }
+
+
+def parse_commandline():
+    parser = OptionParser(version='0.1')
+    parser.add_option('-C', '--config', default="run_tests.conf",
+                      help='The path to the config file to use [%default]')
+    return parser.parse_args()
+
+
+def main():
+    options, arguments = parse_commandline()
+    suite = TestSuite(options.config)
+
+    for path in (p for a in arguments for p in glob.glob(a)):
+        suite.add_test(path)
+
+    suite.run()
+    print suite.get_report().encode('utf-8')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
diff --git a/test/jenkins/run_tests.sh b/test/jenkins/run_tests.sh
new file mode 100755
index 000000000..3ec5ddd80
--- /dev/null
+++ b/test/jenkins/run_tests.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+vagrant ssh-config > ssh_config
+./run_tests.py *.test
+rm -f ssh_config

From 518d0bfd10f5b721abc58021a3e7ab917d53b73b Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Wed, 4 Dec 2013 16:23:44 +0100
Subject: [PATCH 02/17] Fix settings section in configuration file

refs #5223
---
 test/jenkins/run_tests.conf | 4 ++--
 test/jenkins/run_tests.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index 87a7000ad..7b244bc29 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -4,8 +4,8 @@
         "exec": "ssh -F ssh_config default {0}",
         "clean": "ssh -F ssh_config default rm -f {0}"
     },
-    "tests": {
-        "destination": "/tmp"
+    "settings": {
+        "test_root": "/tmp"
     },
     "setups": {
     }
diff --git a/test/jenkins/run_tests.py b/test/jenkins/run_tests.py
index 12612b514..2dea88f9c 100755
--- a/test/jenkins/run_tests.py
+++ b/test/jenkins/run_tests.py
@@ -89,12 +89,12 @@ class TestSuite(object):
         subprocess.call(command, shell=True)
 
     def _copy_test(self, path):
-        self._copy_file(path, os.path.join(self._config['tests']['destination'],
+        self._copy_file(path, os.path.join(self._config['settings']['test_root'],
                                            os.path.basename(path)))
 
     def _run_test(self, path):
         command = self._config['commands']['exec']
-        target = os.path.join(self._config['tests']['destination'],
+        target = os.path.join(self._config['settings']['test_root'],
                               os.path.basename(path))
         p = subprocess.Popen(command.format(target), stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE, shell=True)

From cd05686a7056d77264571839a53396a0210a8d9f Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Wed, 4 Dec 2013 16:33:05 +0100
Subject: [PATCH 03/17] Add state tests

refs #5223
---
 test/jenkins/apache_state.test  | 3 +++
 test/jenkins/icinga2_state.test | 3 +++
 test/jenkins/mysql_state.test   | 3 +++
 test/jenkins/pgsql_state.test   | 3 +++
 test/jenkins/v1.test            | 3 ---
 5 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100755 test/jenkins/apache_state.test
 create mode 100755 test/jenkins/icinga2_state.test
 create mode 100755 test/jenkins/mysql_state.test
 create mode 100755 test/jenkins/pgsql_state.test
 delete mode 100755 test/jenkins/v1.test

diff --git a/test/jenkins/apache_state.test b/test/jenkins/apache_state.test
new file mode 100755
index 000000000..13af22f2f
--- /dev/null
+++ b/test/jenkins/apache_state.test
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo service httpd status
diff --git a/test/jenkins/icinga2_state.test b/test/jenkins/icinga2_state.test
new file mode 100755
index 000000000..b87bd3c56
--- /dev/null
+++ b/test/jenkins/icinga2_state.test
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo service icinga2 status
diff --git a/test/jenkins/mysql_state.test b/test/jenkins/mysql_state.test
new file mode 100755
index 000000000..74608b57f
--- /dev/null
+++ b/test/jenkins/mysql_state.test
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo service mysqld status
diff --git a/test/jenkins/pgsql_state.test b/test/jenkins/pgsql_state.test
new file mode 100755
index 000000000..956ec9cd1
--- /dev/null
+++ b/test/jenkins/pgsql_state.test
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo service postgresql status
diff --git a/test/jenkins/v1.test b/test/jenkins/v1.test
deleted file mode 100755
index f0c0fcd40..000000000
--- a/test/jenkins/v1.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo "Hello World!"
-exit 1

From 3b04b4dcf5c7af13384993b7fd556bd8c6e5fb6d Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Thu, 5 Dec 2013 09:46:51 +0100
Subject: [PATCH 04/17] Add file existence tests

refs #5223
---
 test/jenkins/external_commandpipe.test | 10 +++++++
 test/jenkins/livestatus_socket.test    | 17 ++++++++++++
 test/jenkins/logfile.test              | 10 +++++++
 test/jenkins/pidfile.test              | 10 +++++++
 test/jenkins/statusdata.test           | 37 ++++++++++++++++++++++++++
 5 files changed, 84 insertions(+)
 create mode 100755 test/jenkins/external_commandpipe.test
 create mode 100755 test/jenkins/livestatus_socket.test
 create mode 100755 test/jenkins/logfile.test
 create mode 100755 test/jenkins/pidfile.test
 create mode 100755 test/jenkins/statusdata.test

diff --git a/test/jenkins/external_commandpipe.test b/test/jenkins/external_commandpipe.test
new file mode 100755
index 000000000..5b855e141
--- /dev/null
+++ b/test/jenkins/external_commandpipe.test
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if [ -e "/var/run/icinga2/cmd/icinga2.cmd" ];
+then
+    echo "Icinga2 commandpipe found"
+    exit 0
+else
+    echo "Icinga2 commandpipe not found"
+    exit 1
+fi
diff --git a/test/jenkins/livestatus_socket.test b/test/jenkins/livestatus_socket.test
new file mode 100755
index 000000000..f7b0b4c1a
--- /dev/null
+++ b/test/jenkins/livestatus_socket.test
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ ! -e /var/run/icinga2/cmd/livestatus ];
+then
+    sudo icinga2-enable-feature livestatus 1> /dev/null
+    sudo service icinga2 restart 1> /dev/null
+    sleep 1
+
+    if [ ! -e /var/run/icinga2/cmd/livestatus ];
+    then
+        echo "Icinga2 Livestatus socket not found"
+        exit 1
+    fi
+fi
+
+echo "Icinga2 Livestatus socket found"
+exit 0
diff --git a/test/jenkins/logfile.test b/test/jenkins/logfile.test
new file mode 100755
index 000000000..ae77a56be
--- /dev/null
+++ b/test/jenkins/logfile.test
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if sudo test -f /var/log/icinga2/icinga2.log;
+then
+    echo "Icinga2 log file found"
+    exit 0
+else
+    echo "Icinga2 log file not found"
+    exit 1
+fi
diff --git a/test/jenkins/pidfile.test b/test/jenkins/pidfile.test
new file mode 100755
index 000000000..28c4fde53
--- /dev/null
+++ b/test/jenkins/pidfile.test
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if [ -f /var/run/icinga2/icinga2.pid ];
+then
+    echo "Icinga2 pidfile found"
+    exit 0
+else
+    echo "Icinga2 pidfile not found"
+    exit 1
+fi
diff --git a/test/jenkins/statusdata.test b/test/jenkins/statusdata.test
new file mode 100755
index 000000000..6599795a7
--- /dev/null
+++ b/test/jenkins/statusdata.test
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+if [ ! -f /var/cache/icinga2/status.dat ];
+then
+    sudo icinga2-enable-feature statusdata 1> /dev/null
+    sudo service icinga2 restart 1> /dev/null
+
+    n=0
+    while [ $n -lt 3 ]
+    do
+        sleep 15
+
+        if [ -f /var/cache/icinga2/status.dat ];
+        then
+            break
+        fi
+
+        n=$(( $n + 1))
+    done
+
+    if [ $n -eq 3 ];
+    then
+        echo "Icinga2 status.dat not found"
+        exit 1
+    fi
+fi
+
+echo "Icinga2 status.dat found"
+
+if [ -f /var/cache/icinga2/objects.cache ];
+then
+    echo "Icinga2 objects.cache found"
+    exit 0
+else
+    echo "Icinga2 objects.cache not found"
+    exit 1
+fi

From 17e36fe1cc37ba6cccaffe2b363302adda91b205 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Fri, 6 Dec 2013 10:07:55 +0100
Subject: [PATCH 05/17] Add IDO tests

refs #5223
---
 test/jenkins/files/ido_tests.py | 184 ++++++++++++++++++++++++++++++++
 test/jenkins/ido_mysql.test     |  86 +++++++++++++++
 test/jenkins/ido_pgsql.test     |  91 ++++++++++++++++
 test/jenkins/run_tests.conf     |   8 ++
 4 files changed, 369 insertions(+)
 create mode 100644 test/jenkins/files/ido_tests.py
 create mode 100755 test/jenkins/ido_mysql.test
 create mode 100755 test/jenkins/ido_pgsql.test

diff --git a/test/jenkins/files/ido_tests.py b/test/jenkins/files/ido_tests.py
new file mode 100644
index 000000000..efbd151f1
--- /dev/null
+++ b/test/jenkins/files/ido_tests.py
@@ -0,0 +1,184 @@
+from __future__ import unicode_literals
+
+from datetime import datetime, timedelta
+
+
+TABLE_PREFIX = 'icinga_'
+TABLES = [
+    # Central tables
+    'instances',
+    'objects',
+    # Debugging tables
+    'conninfo',
+    # Historical tables
+    'acknowledgements',
+    'commenthistory',
+    'contactnotifications',
+    'dbversion',
+    'downtimehistory',
+    'eventhandlers',
+    'externalcommands',
+    'flappinghistory',
+    'hostchecks',
+    'logentries',
+    'notifications',
+    'processevents',
+    'servicechecks',
+    'statehistory',
+    'systemcommands',
+    # Current status tables
+    'comments',
+    'customvariablestatus',
+    'hoststatus',
+    'programstatus',
+    'runtimevariables',
+    'scheduleddowntime',
+    'servicestatus',
+    'contactstatus',
+    # Configuration tables
+    'commands',
+    'configfiles',
+    'configfilevariables',
+    'contact_addresses',
+    'contact_notificationcommands',
+    'contactgroup_members',
+    'contactgroups',
+    'contactnotificationmethods',
+    'contacts',
+    'customvariables',
+    'host_contactgroups',
+    'host_contacts',
+    'host_parenthosts',
+    'hostdependencies',
+    'hostescalation_contactgroups',
+    'hostescalation_contacts',
+    'hostescalations',
+    'hostgroup_members',
+    'hostgroups',
+    'hosts',
+    'service_contactgroups',
+    'service_contacts',
+    'servicedependencies',
+    'serviceescalation_contactgroups',
+    'serviceescalation_contacts',
+    'serviceescalations',
+    'servicegroup_members',
+    'servicegroups',
+    'services',
+    'timeperiod_timeranges',
+    'timeperiods'
+    ]
+EXAMPLE_CONFIG = {
+    'localhost': ['disk', 'http', 'icinga', 'load', 'ping4',
+                  'ping6', 'processes', 'ssh', 'users'],
+    'nsca-ng': ['PassiveService1', 'PassiveService2']
+}
+
+
+def validate_tables(tables):
+    """
+    Return whether all tables of the IDO database scheme exist in
+    the given table listing
+
+    """
+    missing = [n for n in TABLES if TABLE_PREFIX + n not in tables]
+    if missing:
+        print 'Some tables are missing in the IDO'
+        print 'Missing tables: ' + ', '.join(missing)
+        return False
+
+    print 'All tables were found in the IDO'
+    return True
+
+
+def verify_host_config(config_data):
+    """
+    Return whether the example hosts exist in the given "hosts" table
+
+    """
+    if len([1 for e in config_data if e['alias'] in EXAMPLE_CONFIG]) == 2:
+        print 'All example hosts are stored in the IDO'
+        return True
+
+    print 'Some example hosts are missing in the IDO'
+    return False
+
+
+def verify_service_config(config_data):
+    """
+    Return whether the example services exist in the given "services" table
+
+    """
+    for hostname, servicename in ((h, s) for h, ss in EXAMPLE_CONFIG.iteritems()
+                                         for s in ss):
+        # Not very efficient, but suitable for just two hosts...
+        if not any(1 for c in config_data
+                     if c['alias'] == hostname and
+                        c['display_name'] == servicename):
+            print 'The config stored in the IDO is missing some services'
+            return False
+
+    print 'The service config stored in the IDO is correct'
+    return True
+
+
+def check_last_host_status_update(check_info):
+    """
+    Return whether the example hosts are checked as scheduled
+
+    """
+    for info in check_info:
+        if info['alias'] == 'localhost':
+            last_check = datetime.fromtimestamp(float(info['last_check']))
+            if datetime.now() - last_check > timedelta(minutes=5, seconds=10):
+                print 'The last status update of host "localhost"' \
+                      ' was more than 5 minutes ago'
+                return False
+        elif info['alias'] == 'nsca-ng':
+            if float(info['last_check']) > 0:
+                print 'The host "nsca-ng" was checked even though' \
+                      ' it should not be actively checked'
+                return False
+
+    print 'The updates of both example hosts are processed as configured'
+    return True
+
+
+def check_last_service_status_update(check_info):
+    """
+    Return whether the example services are checked as scheduled
+
+    """
+    for info in check_info:
+        if info['display_name'] in EXAMPLE_CONFIG.get(info['alias'], []):
+            last_check = datetime.fromtimestamp(float(info['last_check']))
+            if datetime.now() - last_check > timedelta(minutes=5, seconds=10):
+                print 'The last status update of service "{0}" of' \
+                      ' host "{1}" was more than 5 minutes ago' \
+                      ''.format(info['display_name'], info['alias'])
+                return False
+
+    print 'The updates of all example services are processed as configured'
+    return True
+
+
+def check_logentries(logentry_info):
+    """
+    Return whether the given logentry originates from host "localhost"
+    and refers to its very last hard status change
+
+    """
+    if logentry_info and logentry_info[0]['alias'] == 'localhost':
+        entry_time = datetime.fromtimestamp(float(logentry_info[0]['entry_time']))
+        state_time = datetime.fromtimestamp(float(logentry_info[0]['state_time']))
+        if entry_time - state_time > timedelta(seconds=10):
+            print 'The last hard state of host "localhost"' \
+                  ' seems not to have been logged'
+            return False
+    else:
+        print 'No logs found in the IDO for host "localhost"'
+        return False
+
+    print 'The last hard state of host "localhost" was properly logged'
+    return True
+
diff --git a/test/jenkins/ido_mysql.test b/test/jenkins/ido_mysql.test
new file mode 100755
index 000000000..6f9208596
--- /dev/null
+++ b/test/jenkins/ido_mysql.test
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import sys
+import subprocess
+
+import ido_tests
+
+MYSQL = b"/usr/bin/mysql"
+PARAMS = b"-t -D icinga -u icinga --password=icinga -e".split()
+SEPARATOR = '|'
+
+
+def run_query(query):
+    p = subprocess.Popen([MYSQL] + PARAMS + [query.encode('utf-8')],
+                         stdout=subprocess.PIPE)
+    return parse_result([l.decode('utf-8') for l in p.stdout.readlines()])
+
+
+def parse_result(resultset):
+    result, header = [], None
+    for line in (l for l in resultset if SEPARATOR in l):
+        columns = [c.strip() for c in line[1:-3].split(SEPARATOR)]
+        if header is None:
+            header = columns
+        else:
+            result.append(dict((header[i], v) for i, v in enumerate(columns)))
+    return result
+
+
+def main():
+    if not ido_tests.validate_tables([d['Tables_in_icinga']
+                                      for d in run_query('show tables')]):
+        return 1
+
+    host_info = run_query('select * from icinga_hosts')
+    if not ido_tests.verify_host_config(host_info):
+        return 1
+
+    service_info = run_query(
+        'select c2.alias, c1.* from icinga_services as c1 '
+        'inner join icinga_hosts as c2'
+        ' on c1.host_object_id = c2.host_object_id'
+        )
+    if not ido_tests.verify_service_config(service_info):
+        return 1
+
+    hostchecks_data = run_query(
+        'select c.alias, unix_timestamp(s.last_check) as last_check'
+        ' from icinga_hoststatus as s '
+        'inner join icinga_hosts as c'
+        ' on s.host_object_id = c.host_object_id'
+        )
+    if not ido_tests.check_last_host_status_update(hostchecks_data):
+        return 1
+
+    servicechecks_data = run_query(
+        'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
+        ' from icinga_servicestatus as s '
+        'inner join icinga_services as c1'
+        ' on s.service_object_id = c1.service_object_id '
+        'inner join icinga_hosts as c2'
+        ' on c1.host_object_id = c2.host_object_id'
+        )
+    if not ido_tests.check_last_service_status_update(servicechecks_data):
+        return 1
+
+    logentry_info = run_query(
+        'select hosts.alias,'
+        '       max(unix_timestamp(logs.entry_time)) as entry_time,'
+        '       max(unix_timestamp(hist.state_time)) as state_time'
+        ' from icinga_logentries as logs '
+        'inner join icinga_hosts as hosts'
+        ' on logs.object_id = hosts.host_object_id and hosts.alias = "localhost" '
+        'inner join icinga_statehistory as hist'
+        ' on hist.object_id = hosts.host_object_id and hist.state_type = 1'
+        )
+    if not ido_tests.check_logentries(logentry_info):
+        return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
diff --git a/test/jenkins/ido_pgsql.test b/test/jenkins/ido_pgsql.test
new file mode 100755
index 000000000..b03c33a5b
--- /dev/null
+++ b/test/jenkins/ido_pgsql.test
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import sys
+import subprocess
+
+import ido_tests
+
+PSQL = b"/usr/bin/psql"
+PARAMS = b"-nq -U icinga -d icinga -c".split()
+SEPARATOR = '|'
+ENVIRONMENT = {
+    b'PGPASSWORD': b'icinga'
+    }
+
+
+def run_query(query):
+    p = subprocess.Popen([PSQL] + PARAMS + [query.encode('utf-8')],
+                         stdout=subprocess.PIPE, env=ENVIRONMENT)
+    return parse_result([l.decode('utf-8') for l in p.stdout.readlines()])
+
+
+def parse_result(resultset):
+    result, header = [], None
+    for line in (l for l in resultset if SEPARATOR in l):
+        columns = [c.strip() for c in line.split(SEPARATOR)]
+        if header is None:
+            header = columns
+        else:
+            result.append(dict((header[i], v) for i, v in enumerate(columns)))
+    return result
+
+
+def main():
+    if not ido_tests.validate_tables([d['Name'] for d in run_query('\\dt')
+                                                if d['Type'] == 'table']):
+        return 1
+
+    host_info = run_query('select * from icinga_hosts')
+    if not ido_tests.verify_host_config(host_info):
+        return 1
+
+    service_info = run_query(
+        'select c2.alias, c1.* from icinga_services as c1 '
+        'inner join icinga_hosts as c2'
+        ' on c1.host_object_id = c2.host_object_id'
+        )
+    if not ido_tests.verify_service_config(service_info):
+        return 1
+
+    hostchecks_data = run_query(
+        'select c.alias, unix_timestamp(s.last_check) as last_check'
+        ' from icinga_hoststatus as s '
+        'inner join icinga_hosts as c'
+        ' on s.host_object_id = c.host_object_id'
+        )
+    if not ido_tests.check_last_host_status_update(hostchecks_data):
+        return 1
+
+    servicechecks_data = run_query(
+        'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
+        ' from icinga_servicestatus as s '
+        'inner join icinga_services as c1'
+        ' on s.service_object_id = c1.service_object_id '
+        'inner join icinga_hosts as c2'
+        ' on c1.host_object_id = c2.host_object_id'
+        )
+    if not ido_tests.check_last_service_status_update(servicechecks_data):
+        return 1
+
+    logentry_info = run_query(
+        'select hosts.alias,'
+        '       max(unix_timestamp(logs.entry_time)) as entry_time,'
+        '       max(unix_timestamp(hist.state_time)) as state_time'
+        ' from icinga_logentries as logs '
+        'inner join icinga_hosts as hosts'
+        ' on logs.object_id = hosts.host_object_id '
+        'inner join icinga_statehistory as hist'
+        ' on hist.object_id = hosts.host_object_id '
+        "where hosts.alias = 'localhost' and hist.state_type = 1 "
+        'group by hosts.alias'
+        )
+    if not ido_tests.check_logentries(logentry_info):
+        return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index 7b244bc29..c218f1927 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -8,5 +8,13 @@
         "test_root": "/tmp"
     },
     "setups": {
+        "^ido_[a-z]{2}sql.test$": {
+            "setup": {
+                "copy": ["files/ido_tests.py >> /tmp/ido_tests.py"]
+            },
+            "teardown": {
+                "clean": ["/tmp/ido_tests.py", "/tmp/ido_tests.pyc"]
+            }
+        }
     }
 }

From 7f64e58c325dda75008b6e294b102121ef2e4a15 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Fri, 6 Dec 2013 13:02:26 +0100
Subject: [PATCH 06/17] Make setup output quiet

refs #5223
---
 test/jenkins/run_tests.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/jenkins/run_tests.py b/test/jenkins/run_tests.py
index 2dea88f9c..f0a180cd9 100755
--- a/test/jenkins/run_tests.py
+++ b/test/jenkins/run_tests.py
@@ -12,6 +12,12 @@ from optparse import OptionParser
 from xml.dom.minidom import getDOMImplementation
 
 
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'w')
+
+
 class TestSuite(object):
     def __init__(self, configpath):
         self._tests = []
@@ -78,15 +84,15 @@ class TestSuite(object):
 
     def _remove_file(self, path):
         command = self._config['commands']['clean'].format(path)
-        subprocess.call(command, shell=True)
+        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
 
     def _exec_command(self, command):
         command = self._config['commands']['exec'].format(command)
-        subprocess.call(command, shell=True)
+        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
 
     def _copy_file(self, source, destination):
         command = self._config['commands']['copy'].format(source, destination)
-        subprocess.call(command, shell=True)
+        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
 
     def _copy_test(self, path):
         self._copy_file(path, os.path.join(self._config['settings']['test_root'],

From c1d31179ef21970d62a5452df3343fbd76fa3ddf Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Fri, 6 Dec 2013 14:09:04 +0100
Subject: [PATCH 07/17] Fix command execution over SSH

refs #5223
---
 test/jenkins/run_tests.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index c218f1927..864af44e3 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -1,8 +1,8 @@
 {
     "commands": {
         "copy": "scp -qF ssh_config {0} default:{1}",
-        "exec": "ssh -F ssh_config default {0}",
-        "clean": "ssh -F ssh_config default rm -f {0}"
+        "exec": "ssh -F ssh_config default '{0}'",
+        "clean": "ssh -F ssh_config default 'rm -f {0}'"
     },
     "settings": {
         "test_root": "/tmp"

From 45e5d34ec4e14724143c88ed8ce37feadf511991 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Fri, 6 Dec 2013 15:13:45 +0100
Subject: [PATCH 08/17] Let the IDO tests take reschedule actions into account

refs #5223
---
 test/jenkins/files/ido_tests.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/test/jenkins/files/ido_tests.py b/test/jenkins/files/ido_tests.py
index efbd151f1..a6d3131e1 100644
--- a/test/jenkins/files/ido_tests.py
+++ b/test/jenkins/files/ido_tests.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 from datetime import datetime, timedelta
 
 
+CHECK_INTERVAL = 10 # minutes; The actual interval are 5 minutes but as other
+                    # tests might restart Icinga we need to take any
+                    # rescheduling into account
+
 TABLE_PREFIX = 'icinga_'
 TABLES = [
     # Central tables
@@ -130,9 +134,10 @@ def check_last_host_status_update(check_info):
     for info in check_info:
         if info['alias'] == 'localhost':
             last_check = datetime.fromtimestamp(float(info['last_check']))
-            if datetime.now() - last_check > timedelta(minutes=5, seconds=10):
+            if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
+                                                       seconds=10):
                 print 'The last status update of host "localhost"' \
-                      ' was more than 5 minutes ago'
+                      ' was more than {0} minutes ago'.format(CHECK_INTERVAL)
                 return False
         elif info['alias'] == 'nsca-ng':
             if float(info['last_check']) > 0:
@@ -152,10 +157,12 @@ def check_last_service_status_update(check_info):
     for info in check_info:
         if info['display_name'] in EXAMPLE_CONFIG.get(info['alias'], []):
             last_check = datetime.fromtimestamp(float(info['last_check']))
-            if datetime.now() - last_check > timedelta(minutes=5, seconds=10):
+            if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
+                                                       seconds=10):
                 print 'The last status update of service "{0}" of' \
-                      ' host "{1}" was more than 5 minutes ago' \
-                      ''.format(info['display_name'], info['alias'])
+                      ' host "{1}" was more than {2} minutes ago' \
+                      ''.format(info['display_name'], info['alias'],
+                                CHECK_INTERVAL)
                 return False
 
     print 'The updates of all example services are processed as configured'

From 8a9c6c734230c9890b9e0fdabfb4d153dc64f372 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Mon, 9 Dec 2013 16:03:31 +0100
Subject: [PATCH 09/17] Add checkresult tests

refs #5223
---
 test/jenkins/checkresult.test               | 85 +++++++++++++++++++++
 test/jenkins/files/configs/checkresult.conf |  5 ++
 test/jenkins/files/wait_for_ido.sh          | 38 +++++++++
 test/jenkins/ido_mysql.test                 | 10 ++-
 test/jenkins/ido_pgsql.test                 | 10 ++-
 test/jenkins/run_tests.conf                 | 25 ++++++
 6 files changed, 171 insertions(+), 2 deletions(-)
 create mode 100755 test/jenkins/checkresult.test
 create mode 100644 test/jenkins/files/configs/checkresult.conf
 create mode 100755 test/jenkins/files/wait_for_ido.sh

diff --git a/test/jenkins/checkresult.test b/test/jenkins/checkresult.test
new file mode 100755
index 000000000..530145c24
--- /dev/null
+++ b/test/jenkins/checkresult.test
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import os
+import sys
+import time
+
+from ido_mysql import run_query
+
+
+STATE_OK = 0
+TYPE_PASSIVE_CHECK = 1
+
+CHECK_INTERVAL = 300 # seconds
+CHECKRESULT_READ_INTERVAL = 5 # seconds
+CHECKRESULT_LOCATION = '/tmp/icinga2/checkresults'
+CHECKRESULT_TEMPLATE = """
+host_name=%(hostname)s
+service_description=%(servicename)s
+check_type=%(check_type)s
+check_options=0
+scheduled_check=0
+reschedule_check=0
+latency=0
+start_time=%(start_time)s
+finish_time=%(finish_time)s
+early_timeout=0
+exited_ok=%(excited_ok)s
+return_code=%(return_code)s
+output=%(output)s
+"""
+
+
+def main():
+    # We need to wait a bit first as Icinga processes a
+    # checkresult only if its newer than the last check
+    query = 'select unix_timestamp(s.last_check) as last_check ' \
+            'from icinga_servicestatus as s ' \
+            'inner join icinga_services as c ' \
+            'on s.service_object_id = c.service_object_id ' \
+            "where c.display_name = 'PassiveService1'"
+    state_time = float(next(iter(run_query(query)), {}).get('last_check', '0'))
+    if state_time == 0:
+        print '"PassiveService1" seems not to have been checked yet'
+        return 1
+
+    if (state_time + CHECK_INTERVAL) - time.time() < 30:
+        time.sleep(45)
+
+    # Now pass the checkresult in
+    with open(os.path.join(CHECKRESULT_LOCATION, 'cfoobar'), 'w') as f:
+        f.write(CHECKRESULT_TEMPLATE % {
+            'hostname': 'nsca-ng',
+            'servicename': 'PassiveService1',
+            'check_type': TYPE_PASSIVE_CHECK,
+            'start_time': time.time(),
+            'finish_time': time.time(),
+            'excited_ok': '1',
+            'return_code': STATE_OK,
+            'output': 'Passing in CheckResult header files works!'
+            })
+
+    # And notfiy Icinga that the file has been completely written...
+    with open(os.path.join(CHECKRESULT_LOCATION, 'cfoobar.ok'), 'w') as f:
+        pass
+
+    # Lastly check whether the service changed its state
+    time.sleep(CHECKRESULT_READ_INTERVAL * 2)
+
+    query = 'select s.output ' \
+            'from icinga_servicestatus as s ' \
+            'inner join icinga_services as c ' \
+            'on s.service_object_id = c.service_object_id ' \
+            "where c.display_name = 'PassiveService1'"
+    output = next(iter(run_query(query)), {}).get('output', '')
+    if output != 'Passing in CheckResult header files works!':
+        print 'Checkresult header files seem not to be processed properly'
+        return 1
+
+    print 'Checkresult header files are processed properly'
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/test/jenkins/files/configs/checkresult.conf b/test/jenkins/files/configs/checkresult.conf
new file mode 100644
index 000000000..d5cfd93f4
--- /dev/null
+++ b/test/jenkins/files/configs/checkresult.conf
@@ -0,0 +1,5 @@
+library "compat"
+
+object CheckResultReader "reader" {
+    spool_dir = "/tmp/icinga2/checkresults"
+}
diff --git a/test/jenkins/files/wait_for_ido.sh b/test/jenkins/files/wait_for_ido.sh
new file mode 100755
index 000000000..776f4fc44
--- /dev/null
+++ b/test/jenkins/files/wait_for_ido.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+TIMEOUT=30
+
+case $1 in
+    mysql)
+        TYPE='MySQL'
+        CMD='/usr/bin/mysql -t -D icinga -u icinga --password=icinga -e'
+        ;;
+    pgsql)
+        TYPE='PostgreSQL'
+        CMD='/usr/bin/psql -nq -U icinga -d icinga -c'
+        export PGPASSWORD='icinga'
+        ;;
+    *)
+        echo "No IDO type specifier given!"
+        exit 1
+        ;;
+esac
+
+tries=1
+while true
+do
+    out="`$CMD 'select * from icinga_hosts'`"
+
+    if [ $tries -lt $TIMEOUT ] && [ "$out" == "" ];
+    then
+        sleep 1
+        tries=$(($tries + 1))
+    else
+        if [ $tries -eq $TIMEOUT ];
+        then
+            echo "IDO ($TYPE) does not have any hosts or is not responding" >&2
+        fi
+
+        break
+    fi
+done
diff --git a/test/jenkins/ido_mysql.test b/test/jenkins/ido_mysql.test
index 6f9208596..3d1d20a58 100755
--- a/test/jenkins/ido_mysql.test
+++ b/test/jenkins/ido_mysql.test
@@ -4,7 +4,12 @@ from __future__ import unicode_literals
 import sys
 import subprocess
 
-import ido_tests
+try:
+    import ido_tests
+    IDO_TESTS_FOUND = True
+except ImportError:
+    IDO_TESTS_FOUND = False
+
 
 MYSQL = b"/usr/bin/mysql"
 PARAMS = b"-t -D icinga -u icinga --password=icinga -e".split()
@@ -82,5 +87,8 @@ def main():
 
 
 if __name__ == '__main__':
+    if not IDO_TESTS_FOUND:
+        raise
+
     sys.exit(main())
 
diff --git a/test/jenkins/ido_pgsql.test b/test/jenkins/ido_pgsql.test
index b03c33a5b..9011e87d8 100755
--- a/test/jenkins/ido_pgsql.test
+++ b/test/jenkins/ido_pgsql.test
@@ -4,7 +4,12 @@ from __future__ import unicode_literals
 import sys
 import subprocess
 
-import ido_tests
+try:
+    import ido_tests
+    IDO_TESTS_FOUND = True
+except ImportError:
+    IDO_TESTS_FOUND = False
+
 
 PSQL = b"/usr/bin/psql"
 PARAMS = b"-nq -U icinga -d icinga -c".split()
@@ -87,5 +92,8 @@ def main():
 
 
 if __name__ == '__main__':
+    if not IDO_TESTS_FOUND:
+        raise
+
     sys.exit(main())
 
diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index 864af44e3..d257e00d5 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -15,6 +15,31 @@
             "teardown": {
                 "clean": ["/tmp/ido_tests.py", "/tmp/ido_tests.pyc"]
             }
+        },
+        "checkresult.test": {
+            "setup": {
+                "copy": [
+                    "files/configs/checkresult.conf >> /tmp/checkresult.conf",
+                    "files/wait_for_ido.sh >> /tmp/wait_for_ido.sh",
+                    "ido_mysql.test >> /tmp/ido_mysql.py"
+                ],
+                "exec": [
+                    "sudo mv /tmp/checkresult.conf /etc/icinga2/conf.d/",
+                    "mkdir -p -m 0777 /tmp/icinga2/checkresults",
+                    "sudo service icinga2 restart",
+                    "/tmp/wait_for_ido.sh mysql"
+                ]
+            },
+            "teardown": {
+                "clean": ["/tmp/ido_mysql.py*"],
+                "exec": [
+                    "sudo rm /etc/icinga2/conf.d/checkresult.conf",
+                    "sudo service icinga2 restart",
+                    "rmdir /tmp/icinga2/checkresults",
+                    "/tmp/wait_for_ido.sh mysql",
+                    "/tmp/wait_for_ido.sh pgsql && rm /tmp/wait_for_ido.sh"
+                ]
+            }
         }
     }
 }

From 42687a36162026fb771553e06d6aa792593e61ac Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Mon, 9 Dec 2013 16:04:08 +0100
Subject: [PATCH 10/17] Add --output switch to the test runner

refs #5223
---
 test/jenkins/run_tests.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/test/jenkins/run_tests.py b/test/jenkins/run_tests.py
index f0a180cd9..739aacab8 100755
--- a/test/jenkins/run_tests.py
+++ b/test/jenkins/run_tests.py
@@ -84,15 +84,15 @@ class TestSuite(object):
 
     def _remove_file(self, path):
         command = self._config['commands']['clean'].format(path)
-        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
+        subprocess.call(command, stdout=DEVNULL, shell=True)
 
     def _exec_command(self, command):
         command = self._config['commands']['exec'].format(command)
-        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
+        subprocess.call(command, stdout=DEVNULL, shell=True)
 
     def _copy_file(self, source, destination):
         command = self._config['commands']['copy'].format(source, destination)
-        subprocess.call(command, stdout=DEVNULL, stderr=DEVNULL, shell=True)
+        subprocess.call(command, stdout=DEVNULL, shell=True)
 
     def _copy_test(self, path):
         self._copy_file(path, os.path.join(self._config['settings']['test_root'],
@@ -117,6 +117,9 @@ def parse_commandline():
     parser = OptionParser(version='0.1')
     parser.add_option('-C', '--config', default="run_tests.conf",
                       help='The path to the config file to use [%default]')
+    parser.add_option('-O', '--output',
+                      help='The file which to save the test results. '
+                           '(By default this goes to stdout)')
     return parser.parse_args()
 
 
@@ -128,7 +131,14 @@ def main():
         suite.add_test(path)
 
     suite.run()
-    print suite.get_report().encode('utf-8')
+
+    report = suite.get_report()
+    if options.output is None:
+        print report.encode('utf-8')
+    else:
+        with open(options.output, 'w') as f:
+            f.write(report.encode('utf-8'))
+
     return 0
 
 

From a443e4a4fc0676917cd550d91f4e1093c45713b5 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Tue, 10 Dec 2013 09:27:04 +0100
Subject: [PATCH 11/17] Refactor vm bootstrapping

refs #5223
---
 test/jenkins/bootstrap-vm.sh | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/test/jenkins/bootstrap-vm.sh b/test/jenkins/bootstrap-vm.sh
index 30d8f5d31..586938d5d 100755
--- a/test/jenkins/bootstrap-vm.sh
+++ b/test/jenkins/bootstrap-vm.sh
@@ -1,14 +1,29 @@
 #!/bin/sh
-if [ "$1" != "run-by-jenkins" ]; then
-	echo "This script should not be run manually."
+if [ "$1" != "--force" ]; then
+	echo 'This script is NOT intended to be ran by an individual user.' \
+         'If you are not human, pass "--force" as the first option to it!'
 	exit 1
 fi
 
-echo "10.10.27.1 packages.icinga.org" >> /etc/hosts
+if [ $# -lt 3 ]; then
+    echo 'Too few arguments. You need to pass "--force <user> <host>"' \
+         'to run this script.'
+    exit 1
+fi
 
-groupadd vagrant
 
-rmdir /vagrant && ln -s /root/icinga2 /vagrant
-puppet apply --modulepath=/vagrant/.vagrant-puppet/modules /vagrant/.vagrant-puppet/manifests/default.pp
+user=$2
+host=$3
+
+SSH_OPTIONS="-o PasswordAuthentication=no"
+SSH="ssh $SSH_OPTIONS $user@$host"
+
+$SSH "mkdir -p /tmp/icinga2/puppet-stuff"
+scp -r ../../.vagrant-puppet $user@$host:/tmp/icinga2/puppet-stuff
+
+$SSH "groupadd vagrant"
+$SSH "echo '10.10.27.1 packages.icinga.org' >> /etc/hosts"
+$SSH "puppet apply --modulepath=/tmp/icinga2/puppet-stuff/modules" \
+     "             /tmp/icinga2/puppet-stuff/manifests/default.pp"
 
 exit 0

From 6eb169b2153c6723a642f9c9bef20c8bf95e8aea Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Tue, 10 Dec 2013 10:06:51 +0100
Subject: [PATCH 12/17] Fix vm bootstrapping

refs #5223
---
 test/jenkins/bootstrap-vm.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/jenkins/bootstrap-vm.sh b/test/jenkins/bootstrap-vm.sh
index 586938d5d..b12896fc6 100755
--- a/test/jenkins/bootstrap-vm.sh
+++ b/test/jenkins/bootstrap-vm.sh
@@ -1,8 +1,8 @@
 #!/bin/sh
 if [ "$1" != "--force" ]; then
-	echo 'This script is NOT intended to be ran by an individual user.' \
+    echo 'This script is NOT intended to be ran by an individual user.' \
          'If you are not human, pass "--force" as the first option to it!'
-	exit 1
+    exit 1
 fi
 
 if [ $# -lt 3 ]; then
@@ -18,12 +18,12 @@ host=$3
 SSH_OPTIONS="-o PasswordAuthentication=no"
 SSH="ssh $SSH_OPTIONS $user@$host"
 
-$SSH "mkdir -p /tmp/icinga2/puppet-stuff"
-scp -r ../../.vagrant-puppet $user@$host:/tmp/icinga2/puppet-stuff
+$SSH "mkdir /vagrant"
+scp -qr ../../.vagrant-puppet $user@$host:/vagrant
 
 $SSH "groupadd vagrant"
 $SSH "echo '10.10.27.1 packages.icinga.org' >> /etc/hosts"
-$SSH "puppet apply --modulepath=/tmp/icinga2/puppet-stuff/modules" \
-     "             /tmp/icinga2/puppet-stuff/manifests/default.pp"
+$SSH "puppet apply --modulepath=/vagrant/.vagrant-puppet/modules" \
+     "             /vagrant/.vagrant-puppet/manifests/default.pp"
 
 exit 0

From f8887c2f89bc2959f5772d7de6213390ad0775ad Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Tue, 10 Dec 2013 12:25:14 +0100
Subject: [PATCH 13/17] Update readme file

refs #5223
---
 test/jenkins/README | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/jenkins/README b/test/jenkins/README
index ff0681810..78f8a0bef 100644
--- a/test/jenkins/README
+++ b/test/jenkins/README
@@ -8,11 +8,12 @@ This directory contains a few scripts primarily used by build.icinga.org.
   is applied to the VM. (Usually not of interest for the typical user.)
 
 * run_tests.sh
-  This is a wrapper script intended to be ran manually by a user.
+  This is a wrapper script intended to be ran manually by a user. (Note
+  that you need to start this project's vagrant box for this to work!)
 
 * run_tests.py
-  The actual test-runner. Accepts one option (-C|--config) and expects
-  one or more filenames or -patterns that should be run on the VM.
+  The actual test-runner. Accepts two options (-C|--config, -O|--output) and
+  expects one or more filenames or -patterns that should be run on the VM.
 
 * run_tests.conf
   The default configuration file for the test-runner. (Used when running

From d5dc65a7528ac5d66c8734a6bf85ac7ee0cda4f8 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Tue, 10 Dec 2013 12:37:51 +0100
Subject: [PATCH 14/17] Add check to the statusdata test for regular file
 updates

refs #5223
---
 test/jenkins/statusdata.test | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/jenkins/statusdata.test b/test/jenkins/statusdata.test
index 6599795a7..00b2000a9 100755
--- a/test/jenkins/statusdata.test
+++ b/test/jenkins/statusdata.test
@@ -30,8 +30,22 @@ echo "Icinga2 status.dat found"
 if [ -f /var/cache/icinga2/objects.cache ];
 then
     echo "Icinga2 objects.cache found"
-    exit 0
 else
     echo "Icinga2 objects.cache not found"
     exit 1
 fi
+
+status_time=$(stat --format="%Y" /var/cache/icinga2/status.dat)
+
+now=$(date +"%s")
+sleep $(((15 + 5) - ($now - $status_time)))
+
+new_status_time=$(stat --format="%Y" /var/cache/icinga2/status.dat)
+
+if [ $new_status_time -eq $status_time ];
+then
+    echo "Icinga2 status.dat is not being updated"
+    exit 1
+else
+    echo "Icinga2 status.dat is being updated"
+fi

From 02d24200bb14e201b241153a3c8fa53a99012fc8 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Tue, 10 Dec 2013 14:55:33 +0100
Subject: [PATCH 15/17] Refactor IDO query code

refs #5223
---
 test/jenkins/checkresult.test   |  4 +-
 test/jenkins/files/ido_tests.py |  1 -
 test/jenkins/files/utils.py     | 80 +++++++++++++++++++++++++++++++++
 test/jenkins/ido_mysql.test     | 35 ++-------------
 test/jenkins/ido_pgsql.test     | 38 ++--------------
 test/jenkins/run_tests.conf     | 14 ++++--
 6 files changed, 101 insertions(+), 71 deletions(-)
 create mode 100644 test/jenkins/files/utils.py

diff --git a/test/jenkins/checkresult.test b/test/jenkins/checkresult.test
index 530145c24..6cd923f4c 100755
--- a/test/jenkins/checkresult.test
+++ b/test/jenkins/checkresult.test
@@ -5,7 +5,7 @@ import os
 import sys
 import time
 
-from ido_mysql import run_query
+import utils
 
 
 STATE_OK = 0
@@ -32,6 +32,8 @@ output=%(output)s
 
 
 def main():
+    run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
+
     # We need to wait a bit first as Icinga processes a
     # checkresult only if its newer than the last check
     query = 'select unix_timestamp(s.last_check) as last_check ' \
diff --git a/test/jenkins/files/ido_tests.py b/test/jenkins/files/ido_tests.py
index a6d3131e1..e3db1174a 100644
--- a/test/jenkins/files/ido_tests.py
+++ b/test/jenkins/files/ido_tests.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 from datetime import datetime, timedelta
 
-
 CHECK_INTERVAL = 10 # minutes; The actual interval are 5 minutes but as other
                     # tests might restart Icinga we need to take any
                     # rescheduling into account
diff --git a/test/jenkins/files/utils.py b/test/jenkins/files/utils.py
new file mode 100644
index 000000000..25940b1d7
--- /dev/null
+++ b/test/jenkins/files/utils.py
@@ -0,0 +1,80 @@
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+__all__ = ['parse_statusdata', 'run_mysql_query', 'run_pgsql_query']
+
+
+MYSQL_PARAMS = b"-t -D icinga -u icinga --password=icinga -e".split()
+MYSQL_SEPARATOR = '|'
+
+PGSQL_PARAMS = b"-nq -U icinga -d icinga -c".split()
+PGSQL_SEPARATOR = '|'
+PGSQL_ENVIRONMENT = {
+    b'PGPASSWORD': b'icinga'
+    }
+
+
+def parse_statusdata(data, intelligent_cast=True):
+    parsed_data, data_type, type_data = {}, '', {}
+    for line in (l for l in data.split(os.linesep)
+                   if l and not l.startswith('#')):
+        if '{' in line:
+            data_type = line.partition('{')[0].strip()
+        elif '}' in line:
+            parsed_data.setdefault(data_type, []).append(type_data)
+        else:
+            key, _, value = line.partition('=')
+
+            if intelligent_cast:
+                value = _cast_status_value(value)
+
+            type_data[key.strip()] = value
+
+    return parsed_data
+
+
+def _cast_status_value(value):
+    try:
+        return int(value)
+    except ValueError:
+        try:
+            return float(value)
+        except ValueError:
+            return value
+
+
+def run_mysql_query(query, path):
+    p = subprocess.Popen([path] + MYSQL_PARAMS + [query.encode('utf-8')],
+                         stdout=subprocess.PIPE)
+    return _parse_mysql_result([l.decode('utf-8') for l in p.stdout.readlines()])
+
+
+def _parse_mysql_result(resultset):
+    result, header = [], None
+    for line in (l for l in resultset if MYSQL_SEPARATOR in l):
+        columns = [c.strip() for c in line[1:-3].split(MYSQL_SEPARATOR)]
+        if header is None:
+            header = columns
+        else:
+            result.append(dict((header[i], v) for i, v in enumerate(columns)))
+    return result
+
+
+def run_pgsql_query(query, path):
+    p = subprocess.Popen([path] + PGSQL_PARAMS + [query.encode('utf-8')],
+                         stdout=subprocess.PIPE, env=PGSQL_ENVIRONMENT)
+    return _parse_pgsql_result([l.decode('utf-8') for l in p.stdout.readlines()])
+
+
+def _parse_pgsql_result(resultset):
+    result, header = [], None
+    for line in (l for l in resultset if PGSQL_SEPARATOR in l):
+        columns = [c.strip() for c in line.split(PGSQL_SEPARATOR)]
+        if header is None:
+            header = columns
+        else:
+            result.append(dict((header[i], v) for i, v in enumerate(columns)))
+    return result
+
diff --git a/test/jenkins/ido_mysql.test b/test/jenkins/ido_mysql.test
index 3d1d20a58..e1b6613ed 100755
--- a/test/jenkins/ido_mysql.test
+++ b/test/jenkins/ido_mysql.test
@@ -2,38 +2,14 @@
 from __future__ import unicode_literals
 
 import sys
-import subprocess
 
-try:
-    import ido_tests
-    IDO_TESTS_FOUND = True
-except ImportError:
-    IDO_TESTS_FOUND = False
-
-
-MYSQL = b"/usr/bin/mysql"
-PARAMS = b"-t -D icinga -u icinga --password=icinga -e".split()
-SEPARATOR = '|'
-
-
-def run_query(query):
-    p = subprocess.Popen([MYSQL] + PARAMS + [query.encode('utf-8')],
-                         stdout=subprocess.PIPE)
-    return parse_result([l.decode('utf-8') for l in p.stdout.readlines()])
-
-
-def parse_result(resultset):
-    result, header = [], None
-    for line in (l for l in resultset if SEPARATOR in l):
-        columns = [c.strip() for c in line[1:-3].split(SEPARATOR)]
-        if header is None:
-            header = columns
-        else:
-            result.append(dict((header[i], v) for i, v in enumerate(columns)))
-    return result
+import utils
+import ido_tests
 
 
 def main():
+    run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
+
     if not ido_tests.validate_tables([d['Tables_in_icinga']
                                       for d in run_query('show tables')]):
         return 1
@@ -87,8 +63,5 @@ def main():
 
 
 if __name__ == '__main__':
-    if not IDO_TESTS_FOUND:
-        raise
-
     sys.exit(main())
 
diff --git a/test/jenkins/ido_pgsql.test b/test/jenkins/ido_pgsql.test
index 9011e87d8..2d7dae7a2 100755
--- a/test/jenkins/ido_pgsql.test
+++ b/test/jenkins/ido_pgsql.test
@@ -2,41 +2,14 @@
 from __future__ import unicode_literals
 
 import sys
-import subprocess
 
-try:
-    import ido_tests
-    IDO_TESTS_FOUND = True
-except ImportError:
-    IDO_TESTS_FOUND = False
-
-
-PSQL = b"/usr/bin/psql"
-PARAMS = b"-nq -U icinga -d icinga -c".split()
-SEPARATOR = '|'
-ENVIRONMENT = {
-    b'PGPASSWORD': b'icinga'
-    }
-
-
-def run_query(query):
-    p = subprocess.Popen([PSQL] + PARAMS + [query.encode('utf-8')],
-                         stdout=subprocess.PIPE, env=ENVIRONMENT)
-    return parse_result([l.decode('utf-8') for l in p.stdout.readlines()])
-
-
-def parse_result(resultset):
-    result, header = [], None
-    for line in (l for l in resultset if SEPARATOR in l):
-        columns = [c.strip() for c in line.split(SEPARATOR)]
-        if header is None:
-            header = columns
-        else:
-            result.append(dict((header[i], v) for i, v in enumerate(columns)))
-    return result
+import utils
+import ido_tests
 
 
 def main():
+    run_query = lambda q: utils.run_pgsql_query(q, b'/usr/bin/psql')
+
     if not ido_tests.validate_tables([d['Name'] for d in run_query('\\dt')
                                                 if d['Type'] == 'table']):
         return 1
@@ -92,8 +65,5 @@ def main():
 
 
 if __name__ == '__main__':
-    if not IDO_TESTS_FOUND:
-        raise
-
     sys.exit(main())
 
diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index d257e00d5..545659c36 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -10,10 +10,16 @@
     "setups": {
         "^ido_[a-z]{2}sql.test$": {
             "setup": {
-                "copy": ["files/ido_tests.py >> /tmp/ido_tests.py"]
+                "copy": [
+                    "files/ido_tests.py >> /tmp/ido_tests.py",
+                    "files/utils.py >> /tmp/utils.py"
+                ]
             },
             "teardown": {
-                "clean": ["/tmp/ido_tests.py", "/tmp/ido_tests.pyc"]
+                "clean": [
+                    "/tmp/ido_tests.py*",
+                    "/tmp/utils.py*"
+                ]
             }
         },
         "checkresult.test": {
@@ -21,7 +27,7 @@
                 "copy": [
                     "files/configs/checkresult.conf >> /tmp/checkresult.conf",
                     "files/wait_for_ido.sh >> /tmp/wait_for_ido.sh",
-                    "ido_mysql.test >> /tmp/ido_mysql.py"
+                    "files/utils.py >> /tmp/utils.py"
                 ],
                 "exec": [
                     "sudo mv /tmp/checkresult.conf /etc/icinga2/conf.d/",
@@ -31,7 +37,7 @@
                 ]
             },
             "teardown": {
-                "clean": ["/tmp/ido_mysql.py*"],
+                "clean": ["/tmp/utils.py*"],
                 "exec": [
                     "sudo rm /etc/icinga2/conf.d/checkresult.conf",
                     "sudo service icinga2 restart",

From 6832625bcce4eb1d9abcb7b9b7eb92e835b6fa09 Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Thu, 12 Dec 2013 09:04:40 +0100
Subject: [PATCH 16/17] Add external commands test [WIP]

refs #5223
---
 .vagrant-puppet/manifests/default.pp |  5 +++
 test/jenkins/commands_dummy.test     | 40 ++++++++++++++++++++++++
 test/jenkins/files/utils.py          | 46 +++++++++++++++++++++++++++-
 test/jenkins/run_tests.conf          |  9 ++++++
 4 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100755 test/jenkins/commands_dummy.test

diff --git a/.vagrant-puppet/manifests/default.pp b/.vagrant-puppet/manifests/default.pp
index a1becaa7f..9f06d91e9 100644
--- a/.vagrant-puppet/manifests/default.pp
+++ b/.vagrant-puppet/manifests/default.pp
@@ -16,3 +16,8 @@ file { '/etc/motd':
   owner => root,
   group => root
 }
+
+user { 'vagrant':
+  groups  => 'icingacmd',
+  require => Group['icingacmd']
+}
diff --git a/test/jenkins/commands_dummy.test b/test/jenkins/commands_dummy.test
new file mode 100755
index 000000000..9ffdf744a
--- /dev/null
+++ b/test/jenkins/commands_dummy.test
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import sys
+import time
+import random
+
+import utils
+
+
+USERNAME = 'Icinga 2 Admin'
+
+
+def send_command(command):
+    return LIVESTATUS.query('COMMAND [{0}] {1}'.format(int(time.time()), command))
+
+
+def test_host_comments(hostname):
+    print repr(send_command('ADD_HOST_COMMENT;'+hostname+';0;'+USERNAME+';test'))
+    return True
+
+
+def test_service_comments(hostname, servicename):
+    pass
+
+
+def main():
+    failure = False
+
+    # Check whether host comments are properly processed
+    if not test_host_comments('localhost') or not test_host_comments('nsca-ng'):
+        failure = True
+
+    return 1 if failure else 0
+
+
+if __name__ == '__main__':
+    with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
+        sys.exit(main())
+
diff --git a/test/jenkins/files/utils.py b/test/jenkins/files/utils.py
index 25940b1d7..8827ea111 100644
--- a/test/jenkins/files/utils.py
+++ b/test/jenkins/files/utils.py
@@ -1,9 +1,12 @@
 from __future__ import unicode_literals
 
 import os
+import json
+import socket
 import subprocess
 
-__all__ = ['parse_statusdata', 'run_mysql_query', 'run_pgsql_query']
+__all__ = ['parse_statusdata', 'run_mysql_query', 'run_pgsql_query',
+           'LiveStatusSocket']
 
 
 MYSQL_PARAMS = b"-t -D icinga -u icinga --password=icinga -e".split()
@@ -78,3 +81,44 @@ def _parse_pgsql_result(resultset):
             result.append(dict((header[i], v) for i, v in enumerate(columns)))
     return result
 
+
+class LiveStatusSocket(object):
+    options = [
+        'OutputFormat: json',
+        'KeepAlive: on',
+        'ResponseHeader: fixed16'
+        ]
+
+    def __init__(self, path):
+        self.path = path
+
+    def __enter__(self):
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self.close()
+
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(self.path)
+
+    def close(self):
+        self.sock.shutdown(socket.SHUT_RDWR)
+        self.sock.close()
+
+    def query(self, command):
+        full_command = '\n'.join([command] + self.options)
+        self.send(full_command + '\n')
+        return self.recv()
+
+    def send(self, query):
+        print repr(query)
+        self.sock.sendall(query.encode('utf-8'))
+
+    def recv(self):
+        response_header = self.sock.recv(16)
+        response_code = int(response_header[:3])
+        response_length = int(response_header[3:].strip())
+        return json.loads(self.sock.recv(response_length).decode('utf-8'))
+
diff --git a/test/jenkins/run_tests.conf b/test/jenkins/run_tests.conf
index 545659c36..b16faf882 100644
--- a/test/jenkins/run_tests.conf
+++ b/test/jenkins/run_tests.conf
@@ -46,6 +46,15 @@
                     "/tmp/wait_for_ido.sh pgsql && rm /tmp/wait_for_ido.sh"
                 ]
             }
+        },
+        "external_commands.test": {
+            "setup": {
+                "copy": ["files/utils.py >> /tmp/utils.py"]
+            },
+            "teardown": {
+                "clean": ["/tmp/utils.py*"],
+                "exec": ["sudo service icinga2 restart"]
+            }
         }
     }
 }

From ba4286b54989e582498185a9c7a838ef9bc8934f Mon Sep 17 00:00:00 2001
From: Johannes Meyer <johannes.meyer@netways.de>
Date: Mon, 16 Dec 2013 15:39:31 +0100
Subject: [PATCH 17/17] Drop dummy test and fix livestatus utility class

refs #5223
---
 test/jenkins/commands_dummy.test | 40 --------------------------------
 test/jenkins/files/utils.py      | 35 ++++++++++++++++++++++------
 2 files changed, 28 insertions(+), 47 deletions(-)
 delete mode 100755 test/jenkins/commands_dummy.test

diff --git a/test/jenkins/commands_dummy.test b/test/jenkins/commands_dummy.test
deleted file mode 100755
index 9ffdf744a..000000000
--- a/test/jenkins/commands_dummy.test
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-from __future__ import unicode_literals
-
-import sys
-import time
-import random
-
-import utils
-
-
-USERNAME = 'Icinga 2 Admin'
-
-
-def send_command(command):
-    return LIVESTATUS.query('COMMAND [{0}] {1}'.format(int(time.time()), command))
-
-
-def test_host_comments(hostname):
-    print repr(send_command('ADD_HOST_COMMENT;'+hostname+';0;'+USERNAME+';test'))
-    return True
-
-
-def test_service_comments(hostname, servicename):
-    pass
-
-
-def main():
-    failure = False
-
-    # Check whether host comments are properly processed
-    if not test_host_comments('localhost') or not test_host_comments('nsca-ng'):
-        failure = True
-
-    return 1 if failure else 0
-
-
-if __name__ == '__main__':
-    with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
-        sys.exit(main())
-
diff --git a/test/jenkins/files/utils.py b/test/jenkins/files/utils.py
index 8827ea111..fa017a608 100644
--- a/test/jenkins/files/utils.py
+++ b/test/jenkins/files/utils.py
@@ -82,10 +82,14 @@ def _parse_pgsql_result(resultset):
     return result
 
 
+class LiveStatusError(Exception):
+    pass
+
+
 class LiveStatusSocket(object):
     options = [
-        'OutputFormat: json',
         'KeepAlive: on',
+        'OutputFormat: json',
         'ResponseHeader: fixed16'
         ]
 
@@ -108,17 +112,34 @@ class LiveStatusSocket(object):
         self.sock.close()
 
     def query(self, command):
-        full_command = '\n'.join([command] + self.options)
-        self.send(full_command + '\n')
-        return self.recv()
+        self.send(command)
+        statuscode, response = self.recv()
+
+        if statuscode != 200:
+            raise LiveStatusError(statuscode, response)
+
+        return response
 
     def send(self, query):
-        print repr(query)
-        self.sock.sendall(query.encode('utf-8'))
+        full_query = '\n'.join([query] + self.options)
+        self.sock.sendall((full_query + '\n\n').encode('utf-8'))
 
     def recv(self):
+        response = b''
         response_header = self.sock.recv(16)
         response_code = int(response_header[:3])
         response_length = int(response_header[3:].strip())
-        return json.loads(self.sock.recv(response_length).decode('utf-8'))
+
+        if response_length > 0:
+            while len(response) < response_length:
+                response += self.sock.recv(response_length - len(response))
+
+            response = response.decode('utf-8')
+
+            try:
+                response = json.loads(response)
+            except ValueError:
+                pass
+
+        return response_code, response