Merge pull request #8107 from WuerthPhoenix/feature/v1-actions-execute-command-8034-2

WIP implement feature/actions/execute-command

lib/base/atomic.hpp

@@ -4,6 +4,8 @@
 #define ATOMIC_H
 
 #include <atomic>
+#include <type_traits>
+#include <utility>
 
 namespace icinga
 {
@@ -38,6 +40,80 @@ public:
 	}
 };
 
+/**
+ * Wraps T into a std::atomic<T>-like interface.
+ *
+ * @ingroup base
+ */
+template<class T>
+class NotAtomic
+{
+public:
+	inline T load() const
+	{
+		return m_Value;
+	}
+
+	inline void store(T desired)
+	{
+		m_Value = std::move(desired);
+	}
+
+	T m_Value;
+};
+
+/**
+ * Tells whether to use std::atomic<T> or NotAtomic<T>.
+ *
+ * @ingroup base
+ */
+template<class T>
+struct Atomicable
+{
+	// Doesn't work with too old compilers.
+	//static constexpr bool value = std::is_trivially_copyable<T>::value && sizeof(T) <= sizeof(void*);
+	static constexpr bool value = (std::is_fundamental<T>::value || std::is_pointer<T>::value) && sizeof(T) <= sizeof(void*);
+};
+
+/**
+ * Uses either std::atomic<T> or NotAtomic<T> depending on atomicable.
+ *
+ * @ingroup base
+ */
+template<bool atomicable>
+struct AtomicTemplate;
+
+template<>
+struct AtomicTemplate<false>
+{
+	template<class T>
+	struct tmplt
+	{
+		typedef NotAtomic<T> type;
+	};
+};
+
+template<>
+struct AtomicTemplate<true>
+{
+	template<class T>
+	struct tmplt
+	{
+		typedef std::atomic<T> type;
+	};
+};
+
+/**
+ * Uses either std::atomic<T> or NotAtomic<T> depending on T.
+ *
+ * @ingroup base
+ */
+template<class T>
+struct EventuallyAtomic
+{
+	typedef typename AtomicTemplate<Atomicable<T>::value>::template tmplt<T>::type type;
+};
+
 }
 
 #endif /* ATOMIC_H */

lib/base/configobject.ti

@@ -59,10 +59,10 @@ abstract class ConfigObject : ConfigObjectBase < ConfigType
 	[config, no_user_modify] String __name (Name);
 	[config, no_user_modify] String "name" (ShortName) {
 		get {{{
-			if (m_ShortName.IsEmpty())
+			if (m_ShortName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_ShortName;
+				return m_ShortName.m_Value;
 		}}}
 	};
 	[config, no_user_modify] name(Zone) zone (ZoneName);

lib/db_ido/idochecktask.cpp

@@ -20,7 +20,8 @@ REGISTER_FUNCTION_NONCONST(Internal, IdoCheck, &IdoCheckTask::ScriptFunc, "check
 void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
 	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
 {
-	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
+	ServiceState state;
+	CheckCommand::Ptr commandObj = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
 	Value raw_command = commandObj->GetCommandLine();
 
 	Host::Ptr host;
@@ -28,6 +29,9 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	if (service)
 		resolvers.emplace_back("service", service);
 	resolvers.emplace_back("host", host);
@@ -61,25 +65,70 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
 		return;
 
 	if (idoType.IsEmpty()) {
-		cr->SetOutput("Attribute 'ido_type' must be set.");
+		String output = "Attribute 'ido_type' must be set.";
-		cr->SetState(ServiceUnknown);
+		state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	if (idoName.IsEmpty()) {
-		cr->SetOutput("Attribute 'ido_name' must be set.");
+		String output = "Attribute 'ido_name' must be set.";
-		cr->SetState(ServiceUnknown);
+		state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	Type::Ptr type = Type::GetByName(idoType);
 
 	if (!type || !DbConnection::TypeInstance->IsAssignableFrom(type)) {
-		cr->SetOutput("DB IDO type '" + idoType + "' is invalid.");
+		String output = "DB IDO type '" + idoType + "' is invalid.";
-		cr->SetState(ServiceUnknown);
+		state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
@@ -89,34 +138,78 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
 	DbConnection::Ptr conn = static_pointer_cast<DbConnection>(dtype->GetObject(idoName));
 
 	if (!conn) {
-		cr->SetOutput("DB IDO connection '" + idoName + "' does not exist.");
+		String output = "DB IDO connection '" + idoName + "' does not exist.";
-		cr->SetState(ServiceUnknown);
+		state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	double qps = conn->GetQueryCount(60) / 60.0;
 
 	if (conn->IsPaused()) {
-		cr->SetOutput("DB IDO connection is temporarily disabled on this cluster instance.");
+		String output = "DB IDO connection is temporarily disabled on this cluster instance.";
-		cr->SetState(ServiceOK);
+		state = ServiceOK;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	double pendingQueries = conn->GetPendingQueryCount();
 
 	if (!conn->GetConnected()) {
+		String output;
 		if (conn->GetShouldConnect()) {
-			cr->SetOutput("Could not connect to the database server.");
+			output ="Could not connect to the database server.";
-			cr->SetState(ServiceCritical);
+			state = ServiceCritical;
 		} else {
-			cr->SetOutput("Not currently enabled: Another cluster instance is responsible for the IDO database.");
+			output = "Not currently enabled: Another cluster instance is responsible for the IDO database.";
-			cr->SetState(ServiceOK);
+			state = ServiceOK;
 		}
 
-		checkable->ProcessCheckResult(cr);
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
 
+			Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+		} else {
+			cr->SetState(state);
+			cr->SetOutput(output);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
@@ -130,13 +223,13 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
 		    << " Queries per second: " << std::fixed << std::setprecision(3) << qps
 		    << " Pending queries: " << std::fixed << std::setprecision(3) << pendingQueries << ".";
 
-		cr->SetState(ServiceWarning);
+		state = ServiceWarning;
 	} else {
 		msgbuf << "Connected to the database server (Schema version: '" << schema_version << "')."
 		    << " Queries per second: " << std::fixed << std::setprecision(3) << qps
 		    << " Pending queries: " << std::fixed << std::setprecision(3) << pendingQueries << ".";
 
-		cr->SetState(ServiceOK);
+		state = ServiceOK;
 	}
 
 	if (conn->GetEnableHa()) {
@@ -149,34 +242,49 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
 	if (missingQueriesCritical.IsEmpty() && qps < queriesCritical) {
 		msgbuf << " " << qps << " queries/s lower than critical threshold (" << queriesCritical << " queries/s).";
 
-		cr->SetState(ServiceCritical);
+		state= ServiceCritical;
 	} else if (missingQueriesWarning.IsEmpty() && qps < queriesWarning) {
 		msgbuf << " " << qps << " queries/s lower than warning threshold (" << queriesWarning << " queries/s).";
 
-		cr->SetState(ServiceWarning);
+		state = ServiceWarning;
 	}
 
 	if (missingPendingQueriesCritical.IsEmpty() && pendingQueries > pendingQueriesCritical) {
 		msgbuf << " " << pendingQueries << " pending queries greater than critical threshold ("
 		    << pendingQueriesCritical << " queries).";
 
-		cr->SetState(ServiceCritical);
+		state = ServiceCritical;
 	} else if (missingPendingQueriesWarning.IsEmpty() && pendingQueries > pendingQueriesWarning) {
 		msgbuf << " " << pendingQueries << " pending queries greater than warning threshold ("
 		    << pendingQueriesWarning << " queries).";
 
-		cr->SetState(ServiceWarning);
+		state = ServiceWarning;
 	}
 
-	cr->SetOutput(msgbuf.str());
+	String output = msgbuf.str();
 
-	cr->SetPerformanceData(new Array({
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
-		{ new PerfdataValue("queries", qps, false, "", queriesWarning, queriesCritical) },
+		double now = Utility::GetTime();
-		{ new PerfdataValue("queries_1min", conn->GetQueryCount(60)) },
+		ProcessResult pr;
-		{ new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60)) },
+		pr.PID = -1;
-		{ new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60)) },
+		pr.Output = output;
-		{ new PerfdataValue("pending_queries", pendingQueries, false, "", pendingQueriesWarning, pendingQueriesCritical) }
+		pr.ExecutionStart = now;
-	}));
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = state;
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler(commandObj->GetName(), pr);
+	} else {
+		cr->SetState(state);
+		cr->SetOutput(output);
+
+		cr->SetPerformanceData(new Array({
+			{ new PerfdataValue("queries", qps, false, "", queriesWarning, queriesCritical) },
+			{ new PerfdataValue("queries_1min", conn->GetQueryCount(60)) },
+			{ new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60)) },
+			{ new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60)) },
+			{ new PerfdataValue("pending_queries", pendingQueries, false, "", pendingQueriesWarning, pendingQueriesCritical) }
+		}));
+
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/icinga/apiactions.cpp

@@ -619,8 +619,9 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 		endpoint = HttpUtility::GetLastParameter(params, "endpoint");
 
 	MacroProcessor::ResolverList resolvers;
+	Value macros;
 	if (params->Contains("macros")) {
-		Value macros = HttpUtility::GetLastParameter(params, "macros");
+		macros = HttpUtility::GetLastParameter(params, "macros");
 		if (macros.IsObjectType<Dictionary>()) {
 			resolvers.emplace_back("override", macros);
 		}
@@ -667,27 +668,42 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 	);
 
 	CheckResult::Ptr cr = checkable->GetLastCheckResult();
+	if (!cr)
+		cr = new CheckResult();
 
 	/* Check if resolved_command exists and it is of type command_type */
 	Dictionary::Ptr execMacros = new Dictionary();
 
-	MacroResolver::OverrideMacros = execMacros;
+	MacroResolver::OverrideMacros = macros;
 	Defer o ([]() {
 		MacroResolver::OverrideMacros = nullptr;
 	});
 
+	/* Create execution parameters */
+	Dictionary::Ptr execParams = new Dictionary();
+
 	if (command_type == "CheckCommand") {
 		CheckCommand::Ptr cmd = GetSingleObjectByNameUsingPermissions(CheckCommand::GetTypeName(), resolved_command, ActionsHandler::AuthenticatedApiUser);
 		if (!cmd)
 			return ApiActions::CreateResult(404, "Can't find a valid " + command_type + " for '" + resolved_command + "'.");
-		else
+		else {
+			CheckCommand::ExecuteOverride = cmd;
+			Defer resetCheckCommandOverride([]() {
+				CheckCommand::ExecuteOverride = nullptr;
+			});
 			cmd->Execute(checkable, cr, execMacros, false);
+		}
 	} else if (command_type == "EventCommand") {
 		EventCommand::Ptr cmd = GetSingleObjectByNameUsingPermissions(EventCommand::GetTypeName(), resolved_command, ActionsHandler::AuthenticatedApiUser);
 		if (!cmd)
 			return ApiActions::CreateResult(404, "Can't find a valid " + command_type + " for '" + resolved_command + "'.");
-		else
+		else {
+			EventCommand::ExecuteOverride = cmd;
+			Defer resetCheckCommandOverride([]() {
+				EventCommand::ExecuteOverride = nullptr;
+			});
 			cmd->Execute(checkable, execMacros, false);
+		}
 	} else if (command_type == "NotificationCommand") {
 		NotificationCommand::Ptr cmd = GetSingleObjectByNameUsingPermissions(NotificationCommand::GetTypeName(), resolved_command, ActionsHandler::AuthenticatedApiUser);
 		if (!cmd)
@@ -707,6 +723,7 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 			User::Ptr user = GetSingleObjectByNameUsingPermissions(User::GetTypeName(), resolved_user, ActionsHandler::AuthenticatedApiUser);
 			if (!user)
 				return ApiActions::CreateResult(404, "Can't find a valid user for '" + resolved_user + "'.");
+			execParams->Set("user", user->GetName());
 
 			/* Get notification */
 			String notification_string = "";
@@ -722,6 +739,12 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 			Notification::Ptr notification = GetSingleObjectByNameUsingPermissions(Notification::GetTypeName(), resolved_notification, ActionsHandler::AuthenticatedApiUser);
 			if (!notification)
 				return ApiActions::CreateResult(404, "Can't find a valid notification for '" + resolved_notification + "'.");
+			execParams->Set("notification", notification->GetName());
+
+			NotificationCommand::ExecuteOverride = cmd;
+			Defer resetCheckCommandOverride([]() {
+				NotificationCommand::ExecuteOverride = nullptr;
+			});
 
 			cmd->Execute(notification, user, cr, NotificationType::NotificationCustom,
 				ActionsHandler::AuthenticatedApiUser->GetName(), "", execMacros, false);
@@ -761,9 +784,13 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 	MessageOrigin::Ptr origin = new MessageOrigin();
 	listener->RelayMessage(origin, checkable, updateMessage, true);
 
-	/* Create execution parameters */
+	/* Populate execution parameters */
-	Dictionary::Ptr execParams = new Dictionary();
+	if (command_type == "CheckCommand")
-	execParams->Set("command_type", command_type);
+		execParams->Set("command_type", "check_command");
+	else if (command_type == "EventCommand")
+		execParams->Set("command_type", "event_command");
+	else if (command_type == "NotificationCommand")
+		execParams->Set("command_type", "notification_command");
 	execParams->Set("command", resolved_command);
 	execParams->Set("host", host->GetName());
 	if (service)
@@ -779,18 +806,47 @@ Dictionary::Ptr ApiActions::ExecuteCommand(const ConfigObject::Ptr& object,
 	execParams->Set("source", uuid);
 	execParams->Set("deadline", deadline);
 	execParams->Set("macros", execMacros);
+	execParams->Set("endpoint", resolved_endpoint);
 
 	/* Execute command */
 	bool local = endpointPtr == Endpoint::GetLocalEndpoint();
 	if (local) {
 		ClusterEvents::ExecuteCommandAPIHandler(origin, execParams);
 	} else {
+		/* Check if the child endpoints have Icinga version >= 2.13 */
+		Zone::Ptr localZone = Zone::GetLocalZone();
+		for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
+			/* Fetch immediate child zone members */
+			if (zone->GetParent() == localZone && zone->CanAccessObject(endpointPtr->GetZone())) {
+				std::set<Endpoint::Ptr> endpoints = zone->GetEndpoints();
+
+				for (const Endpoint::Ptr& childEndpoint : endpoints) {
+					if (childEndpoint->GetIcingaVersion() < 21300) {
+						/* Update execution */
+						double now = Utility::GetTime();
+						pending_execution->Set("exit", 126);
+						pending_execution->Set("output", "Endpoint '" + childEndpoint->GetName() + "' has version < 2.13.");
+						pending_execution->Set("start", now);
+						pending_execution->Set("end", now);
+						pending_execution->Remove("pending");
+
+						listener->RelayMessage(origin, checkable, updateMessage, true);
+
+						Dictionary::Ptr result = new Dictionary();
+						result->Set("checkable", checkable->GetName());
+						result->Set("execution", uuid);
+						return ApiActions::CreateResult(202, "Accepted", result);
+					}
+				}
+			}
+		}
+
 		Dictionary::Ptr execMessage = new Dictionary();
 		execMessage->Set("jsonrpc", "2.0");
 		execMessage->Set("method", "event::ExecuteCommand");
 		execMessage->Set("params", execParams);
 
-		listener->SyncSendMessage(endpointPtr, execMessage);
+		listener->RelayMessage(origin, endpointPtr->GetZone(), execMessage, true);
 	}
 
 	Dictionary::Ptr result = new Dictionary();

lib/icinga/checkable.cpp

@@ -20,6 +20,7 @@ boost::signals2::signal<void (const Checkable::Ptr&, const String&, double, cons
 boost::signals2::signal<void (const Checkable::Ptr&, double)> Checkable::OnFlappingChange;
 
 static Timer::Ptr l_CheckablesFireSuppressedNotifications;
+thread_local std::function<void(const Value& commandLine, const ProcessResult&)> Checkable::ExecuteCommandProcessFinishedHandler;
 
 void Checkable::StaticInitialize()
 {

lib/icinga/checkable.hpp

@@ -5,6 +5,7 @@
 
 #include "base/atomic.hpp"
 #include "base/timer.hpp"
+#include "base/process.hpp"
 #include "icinga/i2-icinga.hpp"
 #include "icinga/checkable-ti.hpp"
 #include "icinga/timeperiod.hpp"
@@ -14,6 +15,7 @@
 #include "remote/endpoint.hpp"
 #include "remote/messageorigin.hpp"
 #include <cstdint>
+#include <functional>
 
 namespace icinga
 {
@@ -55,6 +57,7 @@ public:
 	DECLARE_OBJECTNAME(Checkable);
 
 	static void StaticInitialize();
+	static thread_local std::function<void(const Value& commandLine, const ProcessResult&)> ExecuteCommandProcessFinishedHandler;
 
 	Checkable();
 

lib/icinga/checkcommand.cpp

@@ -8,6 +8,8 @@ using namespace icinga;
 
 REGISTER_TYPE(CheckCommand);
 
+thread_local CheckCommand::Ptr CheckCommand::ExecuteOverride;
+
 void CheckCommand::Execute(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
 	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
 {

lib/icinga/checkcommand.hpp

@@ -20,6 +20,8 @@ public:
 	DECLARE_OBJECT(CheckCommand);
 	DECLARE_OBJECTNAME(CheckCommand);
 
+	static thread_local CheckCommand::Ptr ExecuteOverride;
+
 	virtual void Execute(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
 		const Dictionary::Ptr& resolvedMacros = nullptr,
 		bool useResolvedMacros = false);

lib/icinga/clusterevents-check.cpp

@@ -75,6 +75,31 @@ void ClusterEvents::EnqueueCheck(const MessageOrigin::Ptr& origin, const Diction
 	}
 }
 
+static void SendEventExecuteCommand(const Dictionary::Ptr& params, long exitStatus, const String& output,
+	double start, double end, const ApiListener::Ptr& listener, const MessageOrigin::Ptr& origin,
+	const Endpoint::Ptr& sourceEndpoint) {
+	Dictionary::Ptr executedParams = new Dictionary();
+	executedParams->Set("execution", params->Get("source"));
+	executedParams->Set("host", params->Get("host"));
+	if (params->Contains("service"))
+		executedParams->Set("service", params->Get("service"));
+	executedParams->Set("exit", exitStatus);
+	executedParams->Set("output", output);
+	executedParams->Set("start", start);
+	executedParams->Set("end", end);
+
+	if (origin->IsLocal()) {
+		ClusterEvents::ExecutedCommandAPIHandler(origin, executedParams);
+	} else {
+		Dictionary::Ptr executedMessage = new Dictionary();
+		executedMessage->Set("jsonrpc", "2.0");
+		executedMessage->Set("method", "event::ExecutedCommand");
+		executedMessage->Set("params", executedParams);
+
+		listener->SyncSendMessage(sourceEndpoint, executedMessage);
+	}
+}
+
 void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) {
 
 	Endpoint::Ptr sourceEndpoint;
@@ -86,43 +111,85 @@ void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, cons
 
 	if (!sourceEndpoint || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) {
 		Log(LogNotice, "ClusterEvents")
-				<< "Discarding 'execute command' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed).";
+			<< "Discarding 'execute command' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed).";
 		return;
 	}
 
 	ApiListener::Ptr listener = ApiListener::GetInstance();
 
 	if (!listener) {
-		Log(LogCritical, "ApiListener", "No instance available.");
+		Log(LogCritical, "ApiListener") << "No instance available.";
 		return;
 	}
 
+	Defer resetExecuteCommandProcessFinishedHandler ([]() {
+		Checkable::ExecuteCommandProcessFinishedHandler = nullptr;
+	});
+
 	if (params->Contains("source")) {
-		Log(LogCritical, "ApiListener", "Not implemented.");
+		String uuid = params->Get("source");
-		return;
+
+		String checkableName = params->Get("host");
+		if (params->Contains("service"))
+			checkableName += "!" + params->Get("service");
+
+		/* Check deadline */
+		double deadline = params->Get("deadline");
+		if (Utility::GetTime() > deadline) {
+			Log(LogNotice, "ApiListener")
+				<< "Discarding 'ExecuteCheckFromQueue' event for checkable '" << checkableName
+				<< "' from '" << origin->FromClient->GetIdentity() << "': Deadline has expired.";
+			return;
+		}
+
+		Checkable::ExecuteCommandProcessFinishedHandler = [checkableName, listener, sourceEndpoint, origin, params] (const Value& commandLine, const ProcessResult& pr) {
+			if (params->Get("command_type") == "check_command") {
+				Checkable::CurrentConcurrentChecks.fetch_sub(1);
+				Checkable::DecreasePendingChecks();
+			}
+
+			if (pr.ExitStatus > 3) {
+				Process::Arguments parguments = Process::PrepareCommand(commandLine);
+				Log(LogWarning, "ApiListener")
+					<< "Command for object '" << checkableName << "' (PID: " << pr.PID
+					<< ", arguments: " << Process::PrettyPrintArguments(parguments) << ") terminated with exit code "
+					<< pr.ExitStatus << ", output: " << pr.Output;
+			}
+
+			SendEventExecuteCommand(params, pr.ExitStatus, pr.Output, pr.ExecutionStart, pr.ExecutionEnd, listener,
+									origin, sourceEndpoint);
+		};
 	}
 
-	if (!listener->GetAcceptCommands()) {
+	if (!listener->GetAcceptCommands() && !origin->IsLocal()) {
 		Log(LogWarning, "ApiListener")
-				<< "Ignoring command. '" << listener->GetName() << "' does not accept commands.";
+			<< "Ignoring command. '" << listener->GetName() << "' does not accept commands.";
 
-		Host::Ptr host = new Host();
+		String output = "Endpoint '" + Endpoint::GetLocalEndpoint()->GetName() + "' does not accept commands.";
-		Dictionary::Ptr attrs = new Dictionary();
 
-		attrs->Set("__name", params->Get("host"));
+		if (params->Contains("source")) {
-		attrs->Set("type", "Host");
+			double now = Utility::GetTime();
-		attrs->Set("enable_active_checks", false);
+			SendEventExecuteCommand(params, 126, output, now, now, listener, origin, sourceEndpoint);
+		} else {
+			Host::Ptr host = new Host();
+			Dictionary::Ptr attrs = new Dictionary();
 
-		Deserialize(host, attrs, false, FAConfig);
+			attrs->Set("__name", params->Get("host"));
+			attrs->Set("type", "Host");
+			attrs->Set("enable_active_checks", false);
 
-		if (params->Contains("service"))
+			Deserialize(host, attrs, false, FAConfig);
-			host->SetExtension("agent_service_name", params->Get("service"));
 
-		CheckResult::Ptr cr = new CheckResult();
+			if (params->Contains("service"))
-		cr->SetState(ServiceUnknown);
+				host->SetExtension("agent_service_name", params->Get("service"));
-		cr->SetOutput("Endpoint '" + Endpoint::GetLocalEndpoint()->GetName() + "' does not accept commands.");
+
-		Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+			CheckResult::Ptr cr = new CheckResult();
-		listener->SyncSendMessage(sourceEndpoint, message);
+			cr->SetState(ServiceUnknown);
+			cr->SetOutput(output);
+
+			Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+			listener->SyncSendMessage(sourceEndpoint, message);
+		}
 
 		return;
 	}
@@ -152,21 +219,44 @@ void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, cons
 
 	if (command_type == "check_command") {
 		if (!CheckCommand::GetByName(command)) {
-			CheckResult::Ptr cr = new CheckResult();
+			ServiceState state = ServiceUnknown;
-			cr->SetState(ServiceUnknown);
+			String output = "Check command '" + command + "' does not exist.";
-			cr->SetOutput("Check command '" + command + "' does not exist.");
+			double now = Utility::GetTime();
-			Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+			
-			listener->SyncSendMessage(sourceEndpoint, message);
+			if (params->Contains("source")) {
+				SendEventExecuteCommand(params, state, output, now, now, listener, origin, sourceEndpoint);
+			} else {
+				CheckResult::Ptr cr = new CheckResult();
+				cr->SetState(state);
+				cr->SetOutput(output);
+				Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+				listener->SyncSendMessage(sourceEndpoint, message);
+			}
 			return;
 		}
 	} else if (command_type == "event_command") {
 		if (!EventCommand::GetByName(command)) {
-			Log(LogWarning, "ClusterEvents")
+			String output = "Event command '" + command + "' does not exist.";
-					<< "Event command '" << command << "' does not exist.";
+			Log(LogWarning, "ClusterEvents") << output;
+
+			if (params->Contains("source")) {
+				double now = Utility::GetTime();
+				SendEventExecuteCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+			}
 			return;
 		}
-	} else
+	} else if (command_type == "notification_command") {
-		return;
+		if (!NotificationCommand::GetByName(command)) {
+			String output = "Notification command '" + command + "' does not exist.";
+			Log(LogWarning, "ClusterEvents") << output;
+
+			if (params->Contains("source")) {
+				double now = Utility::GetTime();
+				SendEventExecuteCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+			}
+			return;
+		}
+	}
 
 	attrs->Set(command_type, params->Get("command"));
 	attrs->Set("command_endpoint", sourceEndpoint->GetName());
@@ -181,25 +271,74 @@ void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, cons
 		try {
 			host->ExecuteRemoteCheck(macros);
 		} catch (const std::exception& ex) {
-			CheckResult::Ptr cr = new CheckResult();
-			cr->SetState(ServiceUnknown);
-
 			String output = "Exception occurred while checking '" + host->GetName() + "': " + DiagnosticInformation(ex);
-			cr->SetOutput(output);
+			ServiceState state = ServiceUnknown;
-
 			double now = Utility::GetTime();
-			cr->SetScheduleStart(now);
-			cr->SetScheduleEnd(now);
-			cr->SetExecutionStart(now);
-			cr->SetExecutionEnd(now);
 
-			Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+			if (params->Contains("source")) {
-			listener->SyncSendMessage(sourceEndpoint, message);
+				SendEventExecuteCommand(params, state, output, now, now, listener, origin, sourceEndpoint);
+			} else {
+				CheckResult::Ptr cr = new CheckResult();
+				cr->SetState(state);
+				cr->SetOutput(output);
+				cr->SetScheduleStart(now);
+				cr->SetScheduleEnd(now);
+				cr->SetExecutionStart(now);
+				cr->SetExecutionEnd(now);
+
+				Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
+				listener->SyncSendMessage(sourceEndpoint, message);
+			}
 
 			Log(LogCritical, "checker", output);
 		}
 	} else if (command_type == "event_command") {
-		host->ExecuteEventHandler(macros, true);
+		try {
+			host->ExecuteEventHandler(macros, true);
+		} catch (const std::exception& ex) {
+			if (params->Contains("source")) {
+				String output = "Exception occurred while executing event command '" + command + "' for '" +
+					host->GetName() + "': " + DiagnosticInformation(ex);
+
+				double now = Utility::GetTime();
+				SendEventExecuteCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+			} else {
+				throw;
+			}
+		}
+	} else if (command_type == "notification_command" && params->Contains("source")) {
+		/* Get user */
+		User::Ptr user = new User();
+		Dictionary::Ptr attrs = new Dictionary();
+		attrs->Set("__name", params->Get("user"));
+		attrs->Set("type", User::GetTypeName());
+
+		Deserialize(user, attrs, false, FAConfig);
+
+		/* Get notification */
+		Notification::Ptr notification = new Notification();
+		attrs->Clear();
+		attrs->Set("__name", params->Get("notification"));
+		attrs->Set("type", Notification::GetTypeName());
+		attrs->Set("command", command);
+
+		Deserialize(notification, attrs, false, FAConfig);
+
+		try {
+			CheckResult::Ptr cr = new CheckResult();
+			String author = macros->Get("notification_author");
+			NotificationCommand::Ptr notificationCommand = NotificationCommand::GetByName(command);
+
+			notificationCommand->Execute(notification, user, cr, NotificationType::NotificationCustom,
+				author, "");
+		} catch (const std::exception& ex) {
+			String output = "Exception occurred during notification '" + notification->GetName()
+				+ "' for checkable '" + notification->GetCheckable()->GetName()
+				+ "' and user '" + user->GetName() + "' using command '" + command + "': "
+				+ DiagnosticInformation(ex, false);
+			double now = Utility::GetTime();
+			SendEventExecuteCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+		}
 	}
 }
 

lib/icinga/clusterevents.cpp

@@ -619,6 +619,63 @@ Value ClusterEvents::AcknowledgementClearedAPIHandler(const MessageOrigin::Ptr&
 
 Value ClusterEvents::ExecuteCommandAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
 {
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+	if (!listener)
+		return Empty;
+
+	if (params->Contains("endpoint")) {
+		Endpoint::Ptr execEndpoint = Endpoint::GetByName(params->Get("endpoint"));
+		if (execEndpoint != Endpoint::GetLocalEndpoint()) {
+
+			Zone::Ptr endpointZone = execEndpoint->GetZone();
+			Zone::Ptr localZone = Zone::GetLocalZone();
+
+			if (!endpointZone->IsChildOf(localZone)) {
+				return Empty;
+			}
+
+			/* Check if the child endpoints have Icinga version >= 2.13 */
+			for (const Zone::Ptr &zone : ConfigType::GetObjectsByType<Zone>()) {
+				/* Fetch immediate child zone members */
+				if (zone->GetParent() == localZone && zone->CanAccessObject(endpointZone)) {
+					std::set<Endpoint::Ptr> endpoints = zone->GetEndpoints();
+
+					for (const Endpoint::Ptr &childEndpoint : endpoints) {
+						if (childEndpoint->GetIcingaVersion() < 21300) {
+							double now = Utility::GetTime();
+							Dictionary::Ptr executedParams = new Dictionary();
+							executedParams->Set("execution", params->Get("source"));
+							executedParams->Set("host", params->Get("host"));
+							if (params->Contains("service"))
+								executedParams->Set("service", params->Get("service"));
+							executedParams->Set("exit", 126);
+							executedParams->Set("output",
+								"Endpoint '" + childEndpoint->GetName() + "' has version < 2.13.");
+							executedParams->Set("start", now);
+							executedParams->Set("end", now);
+
+							Dictionary::Ptr executedMessage = new Dictionary();
+							executedMessage->Set("jsonrpc", "2.0");
+							executedMessage->Set("method", "event::ExecutedCommand");
+							executedMessage->Set("params", executedParams);
+
+							listener->RelayMessage(nullptr, nullptr, executedMessage, true);
+							return Empty;
+						}
+					}
+				}
+			}
+
+			Dictionary::Ptr execMessage = new Dictionary();
+			execMessage->Set("jsonrpc", "2.0");
+			execMessage->Set("method", "event::ExecuteCommand");
+			execMessage->Set("params", params);
+
+			listener->RelayMessage(origin, endpointZone, execMessage, true);
+			return Empty;
+		}
+	}
+
 	EnqueueCheck(origin, params);
 
 	return Empty;
@@ -944,7 +1001,103 @@ Value ClusterEvents::NotificationSentToAllUsersAPIHandler(const MessageOrigin::P
 
 Value ClusterEvents::ExecutedCommandAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
 {
-	return "Not implemented";
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+	if (!listener)
+		return Empty;
+
+	Endpoint::Ptr endpoint;
+	if (origin->FromClient) {
+		endpoint = origin->FromClient->GetEndpoint();
+	} else if (origin->IsLocal()){
+		endpoint = Endpoint::GetLocalEndpoint();
+	}
+
+	if (!endpoint) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'update executions API handler' message from '" << origin->FromClient->GetIdentity()
+			<< "': Invalid endpoint origin (client not allowed).";
+
+		return Empty;
+	}
+
+	Host::Ptr host = Host::GetByName(params->Get("host"));
+	if (!host)
+		return Empty;
+
+	Checkable::Ptr checkable;
+
+	if (params->Contains("service"))
+		checkable = host->GetServiceByShortName(params->Get("service"));
+	else
+		checkable = host;
+
+	if (!checkable)
+		return Empty;
+
+	ObjectLock oLock (checkable);
+
+	if (origin->FromZone && !origin->FromZone->CanAccessObject(checkable)) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'update executions API handler' message for checkable '" << checkable->GetName()
+			<< "' from '" << origin->FromClient->GetIdentity() << "': Unauthorized access.";
+		return Empty;
+	}
+
+	if (!params->Contains("execution")) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'update executions API handler' message for checkable '" << checkable->GetName()
+			<< "' from '" << origin->FromClient->GetIdentity() << "': Execution UUID missing.";
+		return Empty;
+	}
+	String uuid = params->Get("execution");
+
+	Dictionary::Ptr executions = checkable->GetExecutions();
+	if (!executions) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'update executions API handler' message for checkable '" << checkable->GetName()
+			<< "' from '" << origin->FromClient->GetIdentity() << "': Execution '" << uuid << "' missing.";
+		return Empty;
+	}
+
+	Dictionary::Ptr execution = executions->Get(uuid);
+	if (!execution) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'update executions API handler' message for checkable '" << checkable->GetName()
+			<< "' from '" << origin->FromClient->GetIdentity() << "': Execution '" << uuid << "' missing.";
+		return Empty;
+	}
+
+	if (params->Contains("exit"))
+		execution->Set("exit", params->Get("exit"));
+
+	if (params->Contains("output"))
+		execution->Set("output", params->Get("output"));
+
+	if (params->Contains("start"))
+		execution->Set("start", params->Get("start"));
+
+	if (params->Contains("end"))
+		execution->Set("end", params->Get("end"));
+
+	execution->Remove("pending");
+
+	/* Broadcast the update */
+	Dictionary::Ptr executionsToBroadcast = new Dictionary();
+	executionsToBroadcast->Set(uuid, execution);
+	Dictionary::Ptr updateParams = new Dictionary();
+	updateParams->Set("host", host->GetName());
+	if (params->Contains("service"))
+		updateParams->Set("service", params->Get("service"));
+	updateParams->Set("executions", executionsToBroadcast);
+
+	Dictionary::Ptr updateMessage = new Dictionary();
+	updateMessage->Set("jsonrpc", "2.0");
+	updateMessage->Set("method", "event::UpdateExecutions");
+	updateMessage->Set("params", updateParams);
+
+	listener->RelayMessage(nullptr, checkable, updateMessage, true);
+
+	return Empty;
 }
 
 Value ClusterEvents::UpdateExecutionsAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
@@ -988,5 +1141,16 @@ Value ClusterEvents::UpdateExecutionsAPIHandler(const MessageOrigin::Ptr& origin
 	newExecutions->CopyTo(executions);
 	checkable->SetExecutions(executions);
 
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+	if (!listener)
+		return Empty;
+
+	Dictionary::Ptr updateMessage = new Dictionary();
+	updateMessage->Set("jsonrpc", "2.0");
+	updateMessage->Set("method", "event::UpdateExecutions");
+	updateMessage->Set("params", params);
+
+	listener->RelayMessage(origin, Zone::GetLocalZone(), updateMessage, true);
+
 	return Empty;
 }

lib/icinga/eventcommand.cpp

@@ -7,6 +7,8 @@ using namespace icinga;
 
 REGISTER_TYPE(EventCommand);
 
+thread_local EventCommand::Ptr EventCommand::ExecuteOverride;
+
 void EventCommand::Execute(const Checkable::Ptr& checkable,
 	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
 {

lib/icinga/eventcommand.hpp

@@ -20,6 +20,8 @@ public:
 	DECLARE_OBJECT(EventCommand);
 	DECLARE_OBJECTNAME(EventCommand);
 
+	static thread_local EventCommand::Ptr ExecuteOverride;
+
 	virtual void Execute(const Checkable::Ptr& checkable,
 		const Dictionary::Ptr& resolvedMacros = nullptr,
 		bool useResolvedMacros = false);

lib/icinga/host.ti

@@ -21,10 +21,10 @@ class Host : Checkable
 
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 

lib/icinga/hostgroup.ti

@@ -11,10 +11,10 @@ class HostGroup : CustomVarObject
 {
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 

lib/icinga/notificationcommand.cpp

@@ -7,6 +7,8 @@ using namespace icinga;
 
 REGISTER_TYPE(NotificationCommand);
 
+thread_local NotificationCommand::Ptr NotificationCommand::ExecuteOverride;
+
 Dictionary::Ptr NotificationCommand::Execute(const Notification::Ptr& notification,
 	const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationType& type,
 	const String& author, const String& comment, const Dictionary::Ptr& resolvedMacros,

lib/icinga/notificationcommand.hpp

@@ -22,6 +22,8 @@ public:
 	DECLARE_OBJECT(NotificationCommand);
 	DECLARE_OBJECTNAME(NotificationCommand);
 
+	static thread_local NotificationCommand::Ptr ExecuteOverride;
+
 	virtual Dictionary::Ptr Execute(const intrusive_ptr<Notification>& notification,
 		const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationType& type,
 		const String& author, const String& comment,

lib/icinga/service.ti

@@ -33,10 +33,10 @@ class Service : Checkable < ServiceNameComposer
 
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetShortName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 	[config, required] name(Host) host_name;

lib/icinga/servicegroup.ti

@@ -11,10 +11,10 @@ class ServiceGroup : CustomVarObject
 {
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 

lib/icinga/timeperiod.ti

@@ -12,10 +12,10 @@ class TimePeriod : CustomVarObject
 {
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 	[config] Dictionary::Ptr ranges;

lib/icinga/user.ti

@@ -13,10 +13,10 @@ class User : CustomVarObject
 {
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 	[config, no_user_modify, required] array(name(UserGroup)) groups {

lib/icinga/usergroup.ti

@@ -11,10 +11,10 @@ class UserGroup : CustomVarObject
 {
 	[config] String display_name {
 		get {{{
-			if (m_DisplayName.IsEmpty())
+			if (m_DisplayName.m_Value.IsEmpty())
 				return GetName();
 			else
-				return m_DisplayName;
+				return m_DisplayName.m_Value;
 		}}}
 	};
 

lib/methods/clusterchecktask.cpp

@@ -28,46 +28,70 @@ void ClusterCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRe
 	if (resolvedMacros && !useResolvedMacros)
 		return;
 
-	CheckCommand::Ptr command = checkable->GetCheckCommand();
+	CheckCommand::Ptr command = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
-	cr->SetCommand(command->GetName());
+	String commandName = command->GetName();
 
 	ApiListener::Ptr listener = ApiListener::GetInstance();
-
 	if (!listener) {
-		cr->SetOutput("No API listener is configured for this instance.");
+		String output = "No API listener is configured for this instance.";
-		cr->SetState(ServiceUnknown);
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
-		checkable->ProcessCheckResult(cr);
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = 126;
+			pr.Output = output;
+			Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+		} else {
+			cr->SetOutput(output);
+			cr->SetState(ServiceUnknown);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	std::pair<Dictionary::Ptr, Dictionary::Ptr> stats = listener->GetStatus();
-
 	Dictionary::Ptr status = stats.first;
-
-	/* use feature stats perfdata */
-	std::pair<Dictionary::Ptr, Array::Ptr> feature_stats = CIB::GetFeatureStats();
-	cr->SetPerformanceData(feature_stats.second);
-
 	int numConnEndpoints = status->Get("num_conn_endpoints");
 	int numNotConnEndpoints = status->Get("num_not_conn_endpoints");
 
+	ServiceState state;
 	String output = "Icinga 2 Cluster";
 
 	if (numNotConnEndpoints > 0) {
 		output += " Problem: " + Convert::ToString(numNotConnEndpoints) + " endpoints are not connected.";
 		output += "\n(" + FormatArray(status->Get("not_conn_endpoints")) + ")";
 
-		cr->SetState(ServiceCritical);
+		state = ServiceCritical;
 	} else {
 		output += " OK: " + Convert::ToString(numConnEndpoints) + " endpoints are connected.";
 		output += "\n(" + FormatArray(status->Get("conn_endpoints")) + ")";
 
-		cr->SetState(ServiceOK);
+		state = ServiceOK;
 	}
 
-	cr->SetOutput(output);
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		double now = Utility::GetTime();
+		ProcessResult pr;
+		pr.PID = -1;
+		pr.Output = output;
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = state;
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+	} else {
+		/* use feature stats perfdata */
+		std::pair<Dictionary::Ptr, Array::Ptr> feature_stats = CIB::GetFeatureStats();
+		cr->SetPerformanceData(feature_stats.second);
+
+		cr->SetCommand(commandName);
+		cr->SetState(state);
+		cr->SetOutput(output);
+
+		checkable->ProcessCheckResult(cr);
+	}
 }
 
 String ClusterCheckTask::FormatArray(const Array::Ptr& arr)

lib/methods/clusterzonechecktask.cpp

@@ -21,15 +21,32 @@ void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const Che
 	REQUIRE_NOT_NULL(cr);
 
 	ApiListener::Ptr listener = ApiListener::GetInstance();
+	CheckCommand::Ptr command = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
+	String commandName = command->GetName();
 
 	if (!listener) {
-		cr->SetOutput("No API listener is configured for this instance.");
+		String output = "No API listener is configured for this instance.";
-		cr->SetState(ServiceUnknown);
+		ServiceState state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+		} else {
+			cr->SetCommand(commandName);
+			cr->SetOutput(output);
+			cr->SetState(state);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
-	CheckCommand::Ptr command = checkable->GetCheckCommand();
 	Value raw_command = command->GetCommandLine();
 
 	Host::Ptr host;
@@ -37,6 +54,9 @@ void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const Che
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	if (service)
 		resolvers.emplace_back("service", service);
 	resolvers.emplace_back("host", host);
@@ -58,21 +78,51 @@ void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const Che
 	if (resolvedMacros && !useResolvedMacros)
 		return;
 
-	cr->SetCommand(command->GetName());
-
 	if (zoneName.IsEmpty()) {
-		cr->SetOutput("Macro 'cluster_zone' must be set.");
+		String output = "Macro 'cluster_zone' must be set.";
-		cr->SetState(ServiceUnknown);
+		ServiceState state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+		} else {
+			cr->SetCommand(commandName);
+			cr->SetOutput(output);
+			cr->SetState(state);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
 	Zone::Ptr zone = Zone::GetByName(zoneName);
 
 	if (!zone) {
-		cr->SetOutput("Zone '" + zoneName + "' does not exist.");
+		String output = "Zone '" + zoneName + "' does not exist.";
-		cr->SetState(ServiceUnknown);
+		ServiceState state = ServiceUnknown;
-		checkable->ProcessCheckResult(cr);
+
+		if (Checkable::ExecuteCommandProcessFinishedHandler) {
+			double now = Utility::GetTime();
+			ProcessResult pr;
+			pr.PID = -1;
+			pr.Output = output;
+			pr.ExecutionStart = now;
+			pr.ExecutionEnd = now;
+			pr.ExitStatus = state;
+
+			Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+		} else {
+			cr->SetCommand(commandName);
+			cr->SetOutput(output);
+			cr->SetState(state);
+			checkable->ProcessCheckResult(cr);
+		}
 		return;
 	}
 
@@ -107,34 +157,51 @@ void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const Che
 		bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond();
 	}
 
+	ServiceState state;
+	String output;
 	if (connected) {
-		cr->SetState(ServiceOK);
+		state = ServiceOK;
-		cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
+		output = "Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag);
 
 		/* Check whether the thresholds have been resolved and compare them */
 		if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) {
-			cr->SetState(ServiceCritical);
+			state = ServiceCritical;
-			cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+			output = "Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
-				+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical));
+				+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical);
 		} else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) {
-			cr->SetState(ServiceWarning);
+			state = ServiceWarning;
-			cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+			output = "Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
-				+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning));
+				+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning);
 		}
 	} else {
-		cr->SetState(ServiceCritical);
+		state = ServiceCritical;
-		cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
+		output = "Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag);
 	}
 
-	cr->SetPerformanceData(new Array({
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
-		new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
+		double now = Utility::GetTime();
-		new PerfdataValue("last_messages_sent", lastMessageSent),
+		ProcessResult pr;
-		new PerfdataValue("last_messages_received", lastMessageReceived),
+		pr.PID = -1;
-		new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
+		pr.Output = output;
-		new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
+		pr.ExecutionStart = now;
-		new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
+		pr.ExecutionEnd = now;
-		new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
+		pr.ExitStatus = state;
-	}));
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+	} else {
+		cr->SetCommand(commandName);
+		cr->SetState(state);
+		cr->SetOutput(output);
+		cr->SetPerformanceData(new Array({
+			new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
+			new PerfdataValue("last_messages_sent", lastMessageSent),
+			new PerfdataValue("last_messages_received", lastMessageReceived),
+			new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
+			new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
+			new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
+			new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
+		}));
+
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/methods/dummychecktask.cpp

@@ -22,13 +22,16 @@ void DummyCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResu
 	REQUIRE_NOT_NULL(checkable);
 	REQUIRE_NOT_NULL(cr);
 
-	CheckCommand::Ptr command = checkable->GetCheckCommand();
+	CheckCommand::Ptr command = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
 
 	Host::Ptr host;
 	Service::Ptr service;
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	if (service)
 		resolvers.emplace_back("service", service);
 	resolvers.emplace_back("host", host);
@@ -48,14 +51,26 @@ void DummyCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResu
 	std::pair<String, String> co = PluginUtility::ParseCheckOutput(dummyText);
 
 	double now = Utility::GetTime();
+	String commandName = command->GetName();
 
-	cr->SetOutput(co.first);
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
-	cr->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
+		ProcessResult pr;
-	cr->SetState(PluginUtility::ExitStatusToState(dummyState));
+		pr.PID = -1;
-	cr->SetExitStatus(dummyState);
+		pr.Output = dummyText;
-	cr->SetExecutionStart(now);
+		pr.ExecutionStart = now;
-	cr->SetExecutionEnd(now);
+		pr.ExecutionEnd = now;
-	cr->SetCommand(command->GetName());
+		pr.ExitStatus = dummyState;
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+	} else {
+		cr->SetOutput(co.first);
+		cr->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
+		cr->SetState(PluginUtility::ExitStatusToState(dummyState));
+		cr->SetExitStatus(dummyState);
+		cr->SetExecutionStart(now);
+		cr->SetExecutionEnd(now);
+		cr->SetCommand(commandName);
+
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/methods/exceptionchecktask.cpp

@@ -23,5 +23,19 @@ void ExceptionCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const Check
 	if (resolvedMacros && !useResolvedMacros)
 		return;
 
-	BOOST_THROW_EXCEPTION(ScriptError("Test") << boost::errinfo_api_function("Test"));
+	ScriptError scriptError = ScriptError("Test") << boost::errinfo_api_function("Test");
+
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		double now = Utility::GetTime();
+		ProcessResult pr;
+		pr.PID = -1;
+		pr.Output = scriptError.what();
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = 3;
+
+		Checkable::ExecuteCommandProcessFinishedHandler("", pr);
+	} else {
+		BOOST_THROW_EXCEPTION(ScriptError("Test") << boost::errinfo_api_function("Test"));
+	}
 }

lib/methods/icingachecktask.cpp

@@ -26,13 +26,16 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	REQUIRE_NOT_NULL(checkable);
 	REQUIRE_NOT_NULL(cr);
 
-	CheckCommand::Ptr command = checkable->GetCheckCommand();
+	CheckCommand::Ptr command = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
 
 	Host::Ptr host;
 	Service::Ptr service;
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	if (service)
 		resolvers.emplace_back("service", service);
 	resolvers.emplace_back("host", host);
@@ -148,7 +151,7 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	perfdata->Add(new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond));
 
 	cr->SetPerformanceData(perfdata);
-	cr->SetState(ServiceOK);
+	ServiceState state = ServiceOK;
 
 	String appVersion = Application::GetAppVersion();
 
@@ -160,7 +163,7 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 
 	if (lastReloadFailed > 0) {
 		output += "; Last reload attempt failed at " + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", lastReloadFailed);
-		cr->SetState(ServiceWarning);
+		state =ServiceWarning;
 	}
 
 	/* Indicate a warning when the last synced config caused a stage validation error. */
@@ -173,7 +176,7 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 			output += "; Last zone sync stage validation failed at "
 				+ Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", validationResult->Get("ts"));
 
-			cr->SetState(ServiceWarning);
+			state = ServiceWarning;
 		}
 	}
 
@@ -182,11 +185,25 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	/* Return an error if the version is less than specified (optional). */
 	if (missingIcingaMinVersion.IsEmpty() && !icingaMinVersion.IsEmpty() && Utility::CompareVersion(icingaMinVersion, parsedAppVersion) < 0) {
 		output += "; Minimum version " + icingaMinVersion + " is not installed.";
-		cr->SetState(ServiceCritical);
+		state = ServiceCritical;
 	}
 
-	cr->SetOutput(output);
+	String commandName = command->GetName();
-	cr->SetCommand(command->GetName());
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		double now = Utility::GetTime();
+		ProcessResult pr;
+		pr.PID = -1;
+		pr.Output = output;
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = state;
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
+	} else {
+		cr->SetState(state);
+		cr->SetOutput(output);
+		cr->SetCommand(commandName);
+
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/methods/nullchecktask.cpp

@@ -26,12 +26,25 @@ void NullCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResul
 
 	String output = "Hello from ";
 	output += IcingaApplication::GetInstance()->GetNodeName();
+	ServiceState state = ServiceOK;
 
-	cr->SetOutput(output);
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
-	cr->SetPerformanceData(new Array({
+		double now = Utility::GetTime();
-		new PerfdataValue("time", Convert::ToDouble(Utility::GetTime()))
+		ProcessResult pr;
-	}));
+		pr.PID = -1;
-	cr->SetState(ServiceOK);
+		pr.Output = output;
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = state;
 
-	checkable->ProcessCheckResult(cr);
+		Checkable::ExecuteCommandProcessFinishedHandler("", pr);
+	} else {
+		cr->SetOutput(output);
+		cr->SetPerformanceData(new Array({
+			new PerfdataValue("time", Convert::ToDouble(Utility::GetTime()))
+		}));
+		cr->SetState(state);
+
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/methods/nulleventtask.cpp

@@ -11,4 +11,16 @@ REGISTER_FUNCTION_NONCONST(Internal, NullEvent, &NullEventTask::ScriptFunc, "che
 void NullEventTask::ScriptFunc(const Checkable::Ptr& checkable, const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
 {
 	REQUIRE_NOT_NULL(checkable);
+
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		double now = Utility::GetTime();
+		ProcessResult pr;
+		pr.PID = -1;
+		pr.Output = "";
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = 0;
+
+		Checkable::ExecuteCommandProcessFinishedHandler("", pr);
+	}
 }

lib/methods/pluginchecktask.cpp

@@ -22,7 +22,7 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	REQUIRE_NOT_NULL(checkable);
 	REQUIRE_NOT_NULL(cr);
 
-	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
+	CheckCommand::Ptr commandObj = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
 
 	Host::Ptr host;
 	Service::Ptr service;
@@ -43,9 +43,15 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	if (!checkable->GetCheckTimeout().IsEmpty())
 		timeout = checkable->GetCheckTimeout();
 
+
+	std::function<void(const Value& commandLine, const ProcessResult&)> callback;
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		callback = Checkable::ExecuteCommandProcessFinishedHandler;
+	} else {
+		callback = std::bind(&PluginCheckTask::ProcessFinishedHandler, checkable, cr, _1, _2);
+	}
 	PluginUtility::ExecuteCommand(commandObj, checkable, checkable->GetLastCheckResult(),
-		resolvers, resolvedMacros, useResolvedMacros, timeout,
+		resolvers, resolvedMacros, useResolvedMacros, timeout, callback);
-		std::bind(&PluginCheckTask::ProcessFinishedHandler, checkable, cr, _1, _2));
 
 	if (!resolvedMacros || useResolvedMacros) {
 		Checkable::CurrentConcurrentChecks.fetch_add(1);

lib/methods/plugineventtask.cpp

@@ -21,13 +21,16 @@ void PluginEventTask::ScriptFunc(const Checkable::Ptr& checkable,
 {
 	REQUIRE_NOT_NULL(checkable);
 
-	EventCommand::Ptr commandObj = checkable->GetEventCommand();
+	EventCommand::Ptr commandObj = EventCommand::ExecuteOverride ? EventCommand::ExecuteOverride : checkable->GetEventCommand();
 
 	Host::Ptr host;
 	Service::Ptr service;
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	if (service)
 		resolvers.emplace_back("service", service);
 	resolvers.emplace_back("host", host);
@@ -36,9 +39,14 @@ void PluginEventTask::ScriptFunc(const Checkable::Ptr& checkable,
 
 	int timeout = commandObj->GetTimeout();
 
+	std::function<void(const Value& commandLine, const ProcessResult&)> callback;
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		callback = Checkable::ExecuteCommandProcessFinishedHandler;
+	} else {
+		callback = std::bind(&PluginEventTask::ProcessFinishedHandler, checkable, _1, _2);
+	}
 	PluginUtility::ExecuteCommand(commandObj, checkable, checkable->GetLastCheckResult(),
-		resolvers, resolvedMacros, useResolvedMacros, timeout,
+		resolvers, resolvedMacros, useResolvedMacros, timeout, callback);
-		std::bind(&PluginEventTask::ProcessFinishedHandler, checkable, _1, _2));
 }
 
 void PluginEventTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const Value& commandLine, const ProcessResult& pr)

lib/methods/pluginnotificationtask.cpp

@@ -25,7 +25,7 @@ void PluginNotificationTask::ScriptFunc(const Notification::Ptr& notification,
 	REQUIRE_NOT_NULL(notification);
 	REQUIRE_NOT_NULL(user);
 
-	NotificationCommand::Ptr commandObj = notification->GetCommand();
+	NotificationCommand::Ptr commandObj = NotificationCommand::ExecuteOverride ? NotificationCommand::ExecuteOverride : notification->GetCommand();
 
 	auto type = static_cast<NotificationType>(itype);
 
@@ -42,6 +42,9 @@ void PluginNotificationTask::ScriptFunc(const Notification::Ptr& notification,
 	tie(host, service) = GetHostService(checkable);
 
 	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
 	resolvers.emplace_back("user", user);
 	resolvers.emplace_back("notification", notificationExtra);
 	resolvers.emplace_back("notification", notification);
@@ -53,9 +56,14 @@ void PluginNotificationTask::ScriptFunc(const Notification::Ptr& notification,
 
 	int timeout = commandObj->GetTimeout();
 
+	std::function<void(const Value& commandLine, const ProcessResult&)> callback;
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		callback = Checkable::ExecuteCommandProcessFinishedHandler;
+	} else {
+		callback = std::bind(&PluginNotificationTask::ProcessFinishedHandler, checkable, _1, _2);
+	}
 	PluginUtility::ExecuteCommand(commandObj, checkable, cr, resolvers,
-		resolvedMacros, useResolvedMacros, timeout,
+		resolvedMacros, useResolvedMacros, timeout, callback);
-		std::bind(&PluginNotificationTask::ProcessFinishedHandler, checkable, _1, _2));
 }
 
 void PluginNotificationTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const Value& commandLine, const ProcessResult& pr)

lib/methods/randomchecktask.cpp

@@ -31,22 +31,35 @@ void RandomCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 		+ ". Icinga 2 has been running for " + Utility::FormatDuration(uptime)
 		+ ". Version: " + Application::GetAppVersion();
 
-	cr->SetOutput(output);
+	CheckCommand::Ptr command = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
+	String commandName = command->GetName();
+	ServiceState state = static_cast<ServiceState>(Utility::Random() % 4);
 
-	double random = Utility::Random() % 1000;
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
+		double now = Utility::GetTime();
+		ProcessResult pr;
+		pr.PID = -1;
+		pr.Output = output;
+		pr.ExecutionStart = now;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = state;
 
-	cr->SetPerformanceData(new Array({
+		Checkable::ExecuteCommandProcessFinishedHandler(commandName, pr);
-		new PerfdataValue("time", now),
+	} else {
-		new PerfdataValue("value", random),
+		cr->SetOutput(output);
-		new PerfdataValue("value_1m", random * 0.9),
-		new PerfdataValue("value_5m", random * 0.8),
-		new PerfdataValue("uptime", uptime),
-	}));
 
-	cr->SetState(static_cast<ServiceState>(Utility::Random() % 4));
+		double random = Utility::Random() % 1000;
+		cr->SetPerformanceData(new Array({
+			new PerfdataValue("time", now),
+			new PerfdataValue("value", random),
+			new PerfdataValue("value_1m", random * 0.9),
+			new PerfdataValue("value_5m", random * 0.8),
+			new PerfdataValue("uptime", uptime),
+		}));
 
-	CheckCommand::Ptr command = checkable->GetCheckCommand();
+		cr->SetState(state);
-	cr->SetCommand(command->GetName());
+		cr->SetCommand(commandName);
 
-	checkable->ProcessCheckResult(cr);
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/methods/sleepchecktask.cpp

@@ -18,13 +18,16 @@ void SleepCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResu
     REQUIRE_NOT_NULL(checkable);
     REQUIRE_NOT_NULL(cr);
 
-    CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
+	CheckCommand::Ptr commandObj = CheckCommand::ExecuteOverride ? CheckCommand::ExecuteOverride : checkable->GetCheckCommand();
 
     Host::Ptr host;
     Service::Ptr service;
     tie(host, service) = GetHostService(checkable);
 
-    MacroProcessor::ResolverList resolvers;
+	MacroProcessor::ResolverList resolvers;
+	if (MacroResolver::OverrideMacros)
+		resolvers.emplace_back("override", MacroResolver::OverrideMacros);
+
     if (service)
         resolvers.emplace_back("service", service);
     resolvers.emplace_back("host", host);
@@ -42,13 +45,24 @@ void SleepCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResu
     String output = "Slept for " + Convert::ToString(sleepTime) + " seconds.";
 
     double now = Utility::GetTime();
+	CheckCommand::Ptr command = checkable->GetCheckCommand();
+	String commandName = command->GetName();
 
-    cr->SetOutput(output);
+	if (Checkable::ExecuteCommandProcessFinishedHandler) {
-    cr->SetExecutionStart(now);
+		ProcessResult pr;
-    cr->SetExecutionEnd(now);
+		pr.PID = -1;
+		pr.Output = output;
+		pr.ExecutionStart = now - sleepTime;
+		pr.ExecutionEnd = now;
+		pr.ExitStatus = 0;
 
-    CheckCommand::Ptr command = checkable->GetCheckCommand();
+		Checkable::ExecuteCommandProcessFinishedHandler("", pr);
-    cr->SetCommand(command->GetName());
+	} else {
+		cr->SetOutput(output);
+		cr->SetExecutionStart(now);
+		cr->SetExecutionEnd(now);
+		cr->SetCommand(commandName);
 
-    checkable->ProcessCheckResult(cr);
+		checkable->ProcessCheckResult(cr);
+	}
 }

lib/remote/apilistener.cpp

@@ -29,6 +29,8 @@
 #include <boost/asio/spawn.hpp>
 #include <boost/asio/ssl/context.hpp>
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/regex.hpp>
 #include <boost/system/error_code.hpp>
 #include <climits>
 #include <cstdint>
@@ -506,6 +508,20 @@ void ApiListener::NewClientHandler(
 	}
 }
 
+static const auto l_AppVersionInt (([]() -> unsigned long {
+	auto appVersion (Application::GetAppVersion());
+	boost::regex rgx (R"EOF(^v?(\d+)\.(\d+)\.(\d+))EOF");
+	boost::smatch match;
+
+	if (!boost::regex_search(appVersion.GetData(), match, rgx)) {
+		return 0;
+	}
+
+	return 100u * 100u * boost::lexical_cast<unsigned long>(match[1].str())
+		+ 100u * boost::lexical_cast<unsigned long>(match[2].str())
+		+ boost::lexical_cast<unsigned long>(match[3].str());
+})());
+
 /**
  * Processes a new client connection.
  *
@@ -650,7 +666,9 @@ void ApiListener::NewClientHandlerInternal(
 		JsonRpc::SendMessage(client, new Dictionary({
 			{ "jsonrpc", "2.0" },
 			{ "method", "icinga::Hello" },
-			{ "params", new Dictionary() }
+			{ "params", new Dictionary({
+				{ "version", (double)l_AppVersionInt }
+			}) }
 		}), yc);
 
 		client->async_flush(yc);
@@ -683,6 +701,16 @@ void ApiListener::NewClientHandlerInternal(
 		}
 
 		if (firstByte >= '0' && firstByte <= '9') {
+			JsonRpc::SendMessage(client, new Dictionary({
+				{ "jsonrpc", "2.0" },
+				{ "method", "icinga::Hello" },
+				{ "params", new Dictionary({
+					{ "version", (double)l_AppVersionInt }
+				}) }
+			}), yc);
+
+			client->async_flush(yc);
+
 			ctype = ClientJsonRpc;
 		} else {
 			ctype = ClientHttp;
@@ -1607,6 +1635,18 @@ std::set<HttpServerConnection::Ptr> ApiListener::GetHttpClients() const
 
 Value ApiListener::HelloAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
 {
+	if (origin) {
+		auto client (origin->FromClient);
+
+		if (client) {
+			auto endpoint (client->GetEndpoint());
+
+			if (endpoint) {
+				endpoint->SetIcingaVersion((double)params->Get("version"));
+			}
+		}
+	}
+
 	return Empty;
 }
 

lib/remote/endpoint.ti

@@ -21,6 +21,9 @@ class Endpoint : ConfigObject
 
 	[state] Timestamp local_log_position;
 	[state] Timestamp remote_log_position;
+	[state] "unsigned long" icinga_version {
+		default {{{ return 0; }}}
+	};
 
 	[no_user_modify] bool connecting;
 	[no_user_modify] bool syncing;

tools/mkclass/classcompiler.cpp

@@ -797,7 +797,7 @@ void ClassCompiler::HandleClass(const Klass& klass, const ClassDebugInfo&)
 					<< "{" << std::endl;
 
 				if (field.GetAccessor.empty() && !(field.Attributes & FANoStorage))
-					m_Impl << "\t" << "return m_" << field.GetFriendlyName() << ";" << std::endl;
+					m_Impl << "\t" << "return m_" << field.GetFriendlyName() << ".load();" << std::endl;
 				else
 					m_Impl << field.GetAccessor << std::endl;
 
@@ -835,7 +835,7 @@ void ClassCompiler::HandleClass(const Klass& klass, const ClassDebugInfo&)
 
 					
 				if (field.SetAccessor.empty() && !(field.Attributes & FANoStorage))
-					m_Impl << "\t" << "m_" << field.GetFriendlyName() << " = value;" << std::endl;
+					m_Impl << "\t" << "m_" << field.GetFriendlyName() << ".store(value);" << std::endl;
 				else
 					m_Impl << field.SetAccessor << std::endl << std::endl;
 
@@ -1044,7 +1044,7 @@ void ClassCompiler::HandleClass(const Klass& klass, const ClassDebugInfo&)
 			if (field.Attributes & FANoStorage)
 				continue;
 
-			m_Header << "\t" << field.Type.GetRealType() << " m_" << field.GetFriendlyName() << ";" << std::endl;
+			m_Header << "\tEventuallyAtomic<" << field.Type.GetRealType() << ">::type m_" << field.GetFriendlyName() << ";" << std::endl;
 		}
 		
 		/* signal */
@@ -1431,6 +1431,7 @@ void ClassCompiler::CompileStream(const std::string& path, std::istream& input,
 		<< "#include \"base/type.hpp\"" << std::endl
 		<< "#include \"base/value.hpp\"" << std::endl
 		<< "#include \"base/array.hpp\"" << std::endl
+		<< "#include \"base/atomic.hpp\"" << std::endl
 		<< "#include \"base/dictionary.hpp\"" << std::endl
 		<< "#include <boost/signals2.hpp>" << std::endl << std::endl;