Indicate a warning in the 'icinga' check when cluster stage validation failed

- success: clear the last failed attribute
- failed: populate it with the output and current timestamp

This can be used to highlight this in the 'icinga' check task.
Since 2.9 we don't have problems with circular library dependencies
with just one linked binary, therefore it is safe to include libremote
in libmethods here.

lib/methods/icingachecktask.cpp

@@ -8,6 +8,7 @@
 #include "icinga/icingaapplication.hpp"
 #include "icinga/clusterevents.hpp"
 #include "icinga/checkable.hpp"
+#include "remote/apilistener.hpp"
 #include "base/application.hpp"
 #include "base/objectlock.hpp"
 #include "base/utility.hpp"
@@ -157,6 +158,20 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 		cr->SetState(ServiceWarning);
 	}
 
+	/* Indicate a warning when the last synced config caused a stage validation error. */
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+
+	if (listener) {
+		Dictionary::Ptr validationResult = listener->GetLastFailedZonesStageValidation();
+
+		if (validationResult) {
+			output += "; Last zone sync stage validation failed at "
+				+ Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", validationResult->Get("ts"));
+
+			cr->SetState(ServiceWarning);
+		}
+	}
+
 	/* Extract the version number of the running Icinga2 instance.
 	 * We assume that appVersion will allways be something like 'v2.10.1-8-gaebe6da' and we want to extract '2.10.1'.
 	 */

lib/remote/apilistener-filesync.cpp

@@ -414,17 +414,25 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr,
 			Utility::CopyFile(stagePath, currentPath);
 		}
 
-		Application::RequestRestart();
-	} else {
-		Log(LogCritical, "ApiListener")
-			<< "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir()
-			<< "'. Aborting. Logs: '" << logFile << "'";
-
 		ApiListener::Ptr listener = ApiListener::GetInstance();
 
 		if (listener)
-			listener->UpdateLastFailedZonesStageValidation(pr.Output);
+			listener->ClearLastFailedZonesStageValidation();
+
+		Application::RequestRestart();
+
+		return;
 	}
+
+	/* Error case. */
+	Log(LogCritical, "ApiListener")
+		<< "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir()
+		<< "'. Aborting. Logs: '" << logFile << "'";
+
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+
+	if (listener)
+		listener->UpdateLastFailedZonesStageValidation(pr.Output);
 }
 
 void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir,
@@ -464,3 +472,8 @@ void ApiListener::UpdateLastFailedZonesStageValidation(const String& log)
 
 	SetLastFailedZonesStageValidation(lastFailedZonesStageValidation);
 }
+
+void ApiListener::ClearLastFailedZonesStageValidation()
+{
+	SetLastFailedZonesStageValidation(Dictionary::Ptr());
+}

lib/remote/apilistener.hpp

@@ -188,6 +188,7 @@ private:
 		const std::vector<String>& relativePaths);
 
 	void UpdateLastFailedZonesStageValidation(const String& log);
+	void ClearLastFailedZonesStageValidation();
 
 	/* configsync */
 	void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,