Add configuration aware test runner

refs #5223

test/jenkins/README

@@ -1 +1,42 @@
-These scripts are used by build.icinga.org to set up a test VM.
+Set of scripts to set up and test a virtual demo machine
+========================================================
+
+This directory contains a few scripts primarily used by build.icinga.org.
+
+* bootstrap-vm.sh
+  Ensures that all required software is installed and its configuration
+  is applied to the VM. (Usually not of interest for the typical user.)
+
+* run_tests.sh
+  This is a wrapper script intended to be ran manually by a user.
+
+* run_tests.py
+  The actual test-runner. Accepts one option (-C|--config) and expects
+  one or more filenames or -patterns that should be run on the VM.
+
+* run_tests.conf
+  The default configuration file for the test-runner. (Used when running
+  the wrapper script or when no custom configuration file is passed to the
+  test-runner.)
+
+  Format:
+  - commands: This section is mandatory and contains the commands to use.
+  - settings: This section is mandatory and defines settings that are applied to
+              all tests.
+  - setups: This section is optional and contains setup routines that should
+            be ran before (setup) and after (teardown) any matching test is
+            executed. (Note that only one setup can be effective at a time.)
+
+            Example:
+            "^v[1-9]\.test$": {
+                "setup": {
+                    "copy": ["source >> target"], // Files that should be copied.
+                                                  // Note that these files remain
+                                                  // if not removed explicitly
+                    "clean": ["target"], // Files to delete from the system
+                    "exec": ["cmd1", "cmd2"] // Commands to execute on the system
+                },
+                "teardown": {
+                    // The same kind of instructions as above can be added here
+                }
+            }

test/jenkins/run-tests.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-import sys
-from xml.dom.minidom import getDOMImplementation
-from subprocess import Popen, PIPE
-
-impl = getDOMImplementation()
-result = impl.createDocument(None, "testsuite", None)
-testsuite = result.documentElement
-
-for fn in sys.argv[1:]:
-  process = Popen(["./" + fn], stdout=PIPE, stderr=PIPE)
-  (stdoutdata, stderrdata) = process.communicate()
-
-  testcase = result.createElement("testcase")
-  testcase.setAttribute("classname", "vm")
-  testcase.setAttribute("name", fn)
-
-  systemout = result.createElement("system-out")
-  systemout.appendChild(result.createTextNode(stdoutdata))
-  testcase.appendChild(systemout)
-
-  systemerr = result.createElement("system-err")
-  systemerr.appendChild(result.createTextNode(stderrdata))
-  testcase.appendChild(systemerr)
-
-  if process.returncode != 0:
-    failure = result.createElement("failure")
-    failure.setAttribute("type", "returncode")
-    failure.appendChild(result.createTextNode("code: " + str(process.returncode)))
-    testcase.appendChild(failure)
-
-  testsuite.appendChild(testcase)
-
-print result.toxml()

test/jenkins/run_tests.conf

@@ -0,0 +1,12 @@
+{
+    "commands": {
+        "copy": "scp -qF ssh_config {0} default:{1}",
+        "exec": "ssh -F ssh_config default {0}",
+        "clean": "ssh -F ssh_config default rm -f {0}"
+    },
+    "tests": {
+        "destination": "/tmp"
+    },
+    "setups": {
+    }
+}

test/jenkins/run_tests.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+
+import os
+import re
+import sys
+import json
+import glob
+import subprocess
+from optparse import OptionParser
+from xml.dom.minidom import getDOMImplementation
+
+
+class TestSuite(object):
+    def __init__(self, configpath):
+        self._tests = []
+        self._results = {}
+
+        self.load_config(configpath)
+
+    def add_test(self, filepath):
+        self._tests.append(filepath)
+
+    def load_config(self, filepath):
+        with open(filepath) as f:
+            self._config = json.load(f)
+
+    def get_report(self):
+        dom = getDOMImplementation()
+        document = dom.createDocument(None, 'testsuite', None)
+        xml_root = document.documentElement
+
+        for name, info in self._results.iteritems():
+            testresult = document.createElement('testcase')
+            testresult.setAttribute('classname', 'vm')
+            testresult.setAttribute('name', name)
+
+            systemout = document.createElement('system-out')
+            systemout.appendChild(document.createTextNode(info['stdout']))
+            testresult.appendChild(systemout)
+
+            systemerr = document.createElement('system-err')
+            systemerr.appendChild(document.createTextNode(info['stderr']))
+            testresult.appendChild(systemerr)
+
+            if info['returncode'] != 0:
+                failure = document.createElement('failure')
+                failure.setAttribute('type', 'returncode')
+                failure.appendChild(document.createTextNode(
+                    'code: {0}'.format(info['returncode'])))
+                testresult.appendChild(failure)
+
+            xml_root.appendChild(testresult)
+
+        return document.toxml()
+
+    def run(self):
+        for path in self._tests:
+            test_name = os.path.basename(path)
+            self._apply_setup_routines(test_name, 'setup')
+            self._copy_test(path)
+            self._results[test_name] = self._run_test(path)
+            self._apply_setup_routines(test_name, 'teardown')
+
+    def _apply_setup_routines(self, test_name, context):
+        instructions = next((t[1].get(context)
+                             for t in self._config.get('setups', {}).iteritems()
+                             if re.match(t[0], test_name)), None)
+        if instructions is not None:
+            for instruction in instructions.get('copy', []):
+                source, _, destination = instruction.partition('>>')
+                self._copy_file(source.strip(), destination.strip())
+            for filepath in instructions.get('clean', []):
+                self._remove_file(filepath)
+            for command in instructions.get('exec', []):
+                self._exec_command(command)
+
+    def _remove_file(self, path):
+        command = self._config['commands']['clean'].format(path)
+        subprocess.call(command, shell=True)
+
+    def _exec_command(self, command):
+        command = self._config['commands']['exec'].format(command)
+        subprocess.call(command, shell=True)
+
+    def _copy_file(self, source, destination):
+        command = self._config['commands']['copy'].format(source, destination)
+        subprocess.call(command, shell=True)
+
+    def _copy_test(self, path):
+        self._copy_file(path, os.path.join(self._config['tests']['destination'],
+                                           os.path.basename(path)))
+
+    def _run_test(self, path):
+        command = self._config['commands']['exec']
+        target = os.path.join(self._config['tests']['destination'],
+                              os.path.basename(path))
+        p = subprocess.Popen(command.format(target), stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE, shell=True)
+        out, err = p.communicate()
+
+        return {
+            'stdout': out.decode('utf-8'),
+            'stderr': err.decode('utf-8'),
+            'returncode': p.returncode
+            }
+
+
+def parse_commandline():
+    parser = OptionParser(version='0.1')
+    parser.add_option('-C', '--config', default="run_tests.conf",
+                      help='The path to the config file to use [%default]')
+    return parser.parse_args()
+
+
+def main():
+    options, arguments = parse_commandline()
+    suite = TestSuite(options.config)
+
+    for path in (p for a in arguments for p in glob.glob(a)):
+        suite.add_test(path)
+
+    suite.run()
+    print suite.get_report().encode('utf-8')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+

test/jenkins/run_tests.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+vagrant ssh-config > ssh_config
+./run_tests.py *.test
+rm -f ssh_config