Make checks using 'command_endpoint' work inside HA zones

Previously there was no local processing of the executed
check result, which is mandatory inside a HA cluster.

Additionally this patch splits the command execution and
check result processing into more logical parts, executing
local checks, checks on the same command endpoint, and
remote checks.

More details in the referenced issue.

fixes #8249

Signed-off-by: Michael Friedrich <michael.friedrich@netways.de>

AUTHORS

@@ -12,6 +12,7 @@ Gaël Beaudoin <gaboo@gaboo.org>
 Gerd von Egidy <gerd@egidy.de>
 Gunnar Beutner <gunnar.beutner@netways.de>
 Ildar Hizbulin <hizel@vyborg.ru>
+James Pharaoh <james@pharaoh.uk>
 Jan Andres <jan.andres@berenberg.de>
 Jan Wagner <waja@cyconet.org>
 Jason Young <jyoung15@gmail.com>

lib/icinga/apievents.cpp

@@ -1511,9 +1511,9 @@ Value ApiEvents::AcknowledgementClearedAPIHandler(const MessageOrigin& origin, c
 
 Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dictionary::Ptr& params)
 {
-	Endpoint::Ptr endpoint = origin.FromClient->GetEndpoint();
+	Endpoint::Ptr sourceEndpoint = origin.FromClient->GetEndpoint();
 
-	if (!endpoint || (origin.FromZone && !Zone::GetLocalZone()->IsChildOf(origin.FromZone)))
+	if (!sourceEndpoint || (origin.FromZone && !Zone::GetLocalZone()->IsChildOf(origin.FromZone)))
 		return Empty;
 
 	ApiListener::Ptr listener = ApiListener::GetInstance();
@@ -1539,11 +1539,12 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 		cr->SetState(ServiceUnknown);
 		cr->SetOutput("'" + listener->GetName() + "' does not accept commands.");
 		Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-		listener->SyncSendMessage(endpoint, message);
+		listener->SyncSendMessage(sourceEndpoint, message);
 
 		return Empty;
 	}
 
+	/* use a virtual host object for executing the command */
 	Host::Ptr host = new Host();
 	Dictionary::Ptr attrs = new Dictionary();
 
@@ -1559,7 +1560,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 			cr->SetState(ServiceUnknown);
 			cr->SetOutput("Check command '" + command + "' does not exist.");
 			Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-			listener->SyncSendMessage(endpoint, message);
+			listener->SyncSendMessage(sourceEndpoint, message);
 			return Empty;
 		}
 	} else if (command_type == "event_command") {
@@ -1569,7 +1570,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 		return Empty;
 
 	attrs->Set(command_type, params->Get("command"));
-	attrs->Set("command_endpoint", endpoint->GetName());
+	attrs->Set("command_endpoint", sourceEndpoint->GetName());
 
 	Deserialize(host, attrs, false, FAConfig);
 
@@ -1586,7 +1587,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 
 	if (command_type == "check_command") {
 		try {
-			host->ExecuteCheck(macros, true);
+			host->ExecuteRemoteCheck(macros);
 		} catch (const std::exception& ex) {
 			CheckResult::Ptr cr = new CheckResult();
 			cr->SetState(ServiceUnknown);
@@ -1601,7 +1602,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 			cr->SetExecutionEnd(now);
 
 			Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-			listener->SyncSendMessage(endpoint, message);
+			listener->SyncSendMessage(sourceEndpoint, message);
 
 			Log(LogCritical, "checker", output);
 		}

lib/icinga/checkable-check.cpp

@@ -260,15 +260,37 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig
 
 	Endpoint::Ptr command_endpoint = GetCommandEndpoint();
 
-	if (command_endpoint && (Endpoint::GetLocalEndpoint() != command_endpoint) && GetExtension("agent_check")) {
+	if (command_endpoint && GetExtension("agent_check")) {
+		/* agent checks go through the api */
 		ApiListener::Ptr listener = ApiListener::GetInstance();
 
 		if (listener) {
+			/* send message back to its origin */
 			Dictionary::Ptr message = ApiEvents::MakeCheckResultMessage(this, cr);
 			listener->SyncSendMessage(command_endpoint, message);
+
+			/* HA cluster zone nodes must also process the check result locally
+			 * by fetching the real host/service object if existing
+			 */
+			Host::Ptr tempHost;
+			Service::Ptr tempService;
+			tie(tempHost, tempService) = GetHostService(this);
+			Host::Ptr realHost = Host::GetByName(tempHost->GetName());
+			if (realHost) {
+				Value agent_service_name = GetExtension("agent_service_name");
+				if (!agent_service_name.IsEmpty()) {
+					Checkable::Ptr realCheckable;
+					realCheckable = realHost->GetServiceByShortName(agent_service_name);
+					if (realCheckable) {
+						realCheckable->ProcessCheckResult(cr, origin);
+					}
+				}
+			}
+
 		}
 
 		return;
+
 	}
 
 	bool reachable = IsReachable();
@@ -498,7 +520,21 @@ bool Checkable::IsCheckPending(void) const
 	return m_CheckRunning;
 }
 
-void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
+void Checkable::ExecuteRemoteCheck(const Dictionary::Ptr& resolvedMacros)
+{
+	CONTEXT("Executing remote check for object '" + GetName() + "'");
+
+	double scheduled_start = GetNextCheck();
+	double before_check = Utility::GetTime();
+
+	CheckResult::Ptr cr = new CheckResult();
+	cr->SetScheduleStart(scheduled_start);
+	cr->SetExecutionStart(before_check);
+
+	GetCheckCommand()->Execute(this, cr, resolvedMacros, true);
+}
+
+void Checkable::ExecuteCheck()
 {
 	CONTEXT("Executing check for object '" + GetName() + "'");
 
@@ -526,23 +562,22 @@ void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useReso
 	double scheduled_start = GetNextCheck();
 	double before_check = Utility::GetTime();
 
-	CheckResult::Ptr result = new CheckResult();
+	CheckResult::Ptr cr = new CheckResult();
 
-	result->SetScheduleStart(scheduled_start);
+	cr->SetScheduleStart(scheduled_start);
-	result->SetExecutionStart(before_check);
+	cr->SetExecutionStart(before_check);
 
-	Dictionary::Ptr macros;
 	Endpoint::Ptr endpoint = GetCommandEndpoint();
+	bool local = !endpoint || endpoint == Endpoint::GetLocalEndpoint();
 
-	if (endpoint && !useResolvedMacros)
+	if (local) {
-		macros = new Dictionary();
+		GetCheckCommand()->Execute(this, cr, NULL, false);
-	else
+	} else {
-		macros = resolvedMacros;
+		Dictionary::Ptr macros = new Dictionary();
+		GetCheckCommand()->Execute(this, cr, macros, false);
 
-	GetCheckCommand()->Execute(this, result, macros, useResolvedMacros);
-
-	if (endpoint && !useResolvedMacros) {
 		if (endpoint->IsConnected()) {
+			/* perform check on remote endpoint */
 			Dictionary::Ptr message = new Dictionary();
 			message->Set("jsonrpc", "2.0");
 			message->Set("method", "event::ExecuteCommand");
@@ -566,10 +601,15 @@ void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useReso
 
 			if (listener)
 				listener->SyncSendMessage(endpoint, message);
+
 		} else if (Application::GetInstance()->GetStartTime() < Utility::GetTime() - 30) {
-			result->SetState(ServiceUnknown);
+			/* fail to perform check on unconnected endpoint */
-			result->SetOutput("Remote Icinga instance '" + endpoint->GetName() + "' is not connected.");
+			cr->SetState(ServiceUnknown);
-			ProcessCheckResult(result);
+
+			cr->SetOutput("Remote Icinga instance '" + endpoint->GetName() +
+			    "' " + "is not connected to '" + Endpoint::GetLocalEndpoint()->GetName() + "'");
+
+			ProcessCheckResult(cr);
 		}
 
 		{

lib/icinga/checkable.hpp

@@ -135,8 +135,8 @@ public:
 
 	static void UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type);
 
-	void ExecuteCheck(const Dictionary::Ptr& resolvedMacros = Dictionary::Ptr(),
+	void ExecuteRemoteCheck(const Dictionary::Ptr& resolvedMacros = Dictionary::Ptr());
-	    bool useResolvedMacros = false);
+	void ExecuteCheck();
 	void ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin& origin = MessageOrigin());
 
 	int GetModifiedAttributes(void) const;