Stash notifications until object authority has been updated once

refs #7086

lib/icinga/checkable-notification.cpp

@@ -4,11 +4,13 @@
 #include "icinga/host.hpp"
 #include "icinga/icingaapplication.hpp"
 #include "icinga/service.hpp"
+#include "base/dictionary.hpp"
 #include "base/objectlock.hpp"
 #include "base/logger.hpp"
 #include "base/exception.hpp"
 #include "base/context.hpp"
 #include "base/convert.hpp"
+#include "remote/apilistener.hpp"
 
 using namespace icinga;
 
@@ -54,17 +56,47 @@ void Checkable::SendNotifications(NotificationType type, const CheckResult::Ptr&
 		return;
 
 	for (const Notification::Ptr& notification : notifications) {
-		try {
-			if (!notification->IsPaused()) {
-				notification->BeginExecuteNotification(type, cr, force, false, author, text);
-			} else {
-				Log(LogNotice, "Notification")
-					<< "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+		if (ApiListener::UpdatedObjectAuthority()) {
+			try {
+				if (!notification->IsPaused()) {
+					auto stashedNotifications (notification->GetStashedNotifications());
+
+					if (stashedNotifications->GetLength()) {
+						Log(LogNotice, "Notification")
+							<< "Notification '" << notification->GetName() << "': there are some stashed notifications. Stashing notification to preserve order.";
+
+						stashedNotifications->Add(new Dictionary({
+							{"type", type},
+							{"cr", cr},
+							{"force", force},
+							{"reminder", false},
+							{"author", author},
+							{"text", text}
+						}));
+					} else {
+						notification->BeginExecuteNotification(type, cr, force, false, author, text);
+					}
+				} else {
+					Log(LogNotice, "Notification")
+						<< "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+				}
+			} catch (const std::exception& ex) {
+				Log(LogWarning, "Checkable")
+					<< "Exception occurred during notification '" << notification->GetName() << "' for checkable '"
+					<< GetName() << "': " << DiagnosticInformation(ex, false);
 			}
-		} catch (const std::exception& ex) {
-			Log(LogWarning, "Checkable")
-				<< "Exception occurred during notification '" << notification->GetName() << "' for checkable '"
-				<< GetName() << "': " << DiagnosticInformation(ex, false);
+		} else {
+			Log(LogNotice, "Notification")
+				<< "Notification '" << notification->GetName() << "': object authority hasn't been updated, yet. Stashing notification.";
+
+			notification->GetStashedNotifications()->Add(new Dictionary({
+				{"type", type},
+				{"cr", cr},
+				{"force", force},
+				{"reminder", false},
+				{"author", author},
+				{"text", text}
+			}));
 		}
 	}
 }

lib/icinga/notification.ti

@@ -80,6 +80,10 @@ class Notification : CustomVarObject < NotificationNameComposer
 		default {{{ return false; }}}
 	};
 
+	[state, no_user_view, no_user_modify] Array::Ptr stashed_notifications {
+		default {{{ return new Array(); }}}
+	};
+
 	[state] Timestamp last_notification;
 	[state] Timestamp next_notification;
 	[state] int notification_number;

lib/notification/notificationcomponent.cpp

@@ -10,6 +10,7 @@
 #include "base/utility.hpp"
 #include "base/exception.hpp"
 #include "base/statsfunction.hpp"
+#include "remote/apilistener.hpp"
 
 using namespace icinga;
 
@@ -72,12 +73,27 @@ void NotificationComponent::NotificationTimerHandler()
 			continue;
 
 		String notificationName = notification->GetName();
+		bool updatedObjectAuthority = ApiListener::UpdatedObjectAuthority();
 
 		/* Skip notification if paused, in a cluster setup & HA feature is enabled. */
-		if (notification->IsPaused() && myEndpoint && GetEnableHA()) {
-			Log(LogNotice, "NotificationComponent")
-				<< "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
-			continue;
+		if (notification->IsPaused()) {
+			if (updatedObjectAuthority) {
+				auto stashedNotifications (notification->GetStashedNotifications());
+				ObjectLock olock(stashedNotifications);
+
+				if (stashedNotifications->GetLength()) {
+					Log(LogNotice, "NotificationComponent")
+						<< "Notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority. Dropping all stashed notifications.";
+
+					stashedNotifications->Clear();
+				}
+			}
+
+			if (myEndpoint && GetEnableHA()) {
+				Log(LogNotice, "NotificationComponent")
+					<< "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+				continue;
+			}
 		}
 
 		Checkable::Ptr checkable = notification->GetCheckable();
@@ -85,6 +101,42 @@ void NotificationComponent::NotificationTimerHandler()
 		if (!IcingaApplication::GetInstance()->GetEnableNotifications() || !checkable->GetEnableNotifications())
 			continue;
 
+		bool reachable = checkable->IsReachable(DependencyNotification);
+
+		if (reachable) {
+			Array::Ptr unstashedNotifications = new Array();
+
+			{
+				auto stashedNotifications (notification->GetStashedNotifications());
+				ObjectLock olock(stashedNotifications);
+
+				stashedNotifications->CopyTo(unstashedNotifications);
+				stashedNotifications->Clear();
+			}
+
+			ObjectLock olock(unstashedNotifications);
+
+			for (Dictionary::Ptr unstashedNotification : unstashedNotifications) {
+				try {
+					Log(LogNotice, "NotificationComponent")
+						<< "Attempting to send stashed notification '" << notificationName << "'.";
+
+					notification->BeginExecuteNotification(
+						(NotificationType)(int)unstashedNotification->Get("type"),
+						(CheckResult::Ptr)unstashedNotification->Get("cr"),
+						(bool)unstashedNotification->Get("force"),
+						(bool)unstashedNotification->Get("reminder"),
+						(String)unstashedNotification->Get("author"),
+						(String)unstashedNotification->Get("text")
+					);
+				} catch (const std::exception& ex) {
+					Log(LogWarning, "NotificationComponent")
+						<< "Exception occurred during notification for object '"
+						<< notificationName << "': " << DiagnosticInformation(ex, false);
+				}
+			}
+		}
+
 		if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications()) {
 			Log(LogNotice, "NotificationComponent")
 				<< "Reminder notification '" << notificationName << "': Notification was sent out once and interval=0 disables reminder notifications.";
@@ -94,8 +146,6 @@ void NotificationComponent::NotificationTimerHandler()
 		if (notification->GetNextNotification() > now)
 			continue;
 
-		bool reachable = checkable->IsReachable(DependencyNotification);
-
 		{
 			ObjectLock olock(notification);
 			notification->SetNextNotification(Utility::GetTime() + notification->GetInterval());