Ensure that sent notifications are synced between Icinga instances

fixes #11562

lib/icinga/apievents.cpp

@@ -107,7 +107,7 @@ void ApiEvents::StateChangeHandler(const Checkable::Ptr& checkable, const CheckR
 
 void ApiEvents::NotificationSentToAllUsersHandler(const Notification::Ptr& notification,
     const Checkable::Ptr& checkable, const std::set<User::Ptr>& users, NotificationType type,
-    const CheckResult::Ptr& cr, const String& author, const String& text)
+    const CheckResult::Ptr& cr, const String& author, const String& text, const MessageOrigin::Ptr& origin)
 {
 	std::vector<EventQueue::Ptr> queues = EventQueue::GetQueuesForType("Notification");
 

lib/icinga/apievents.hpp

@@ -40,7 +40,7 @@ public:
 
 	static void NotificationSentToAllUsersHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
 	    const std::set<User::Ptr>& users, NotificationType type, const CheckResult::Ptr& cr, const String& author,
-	    const String& text);
+	    const String& text, const MessageOrigin::Ptr& origin);
 
 	static void FlappingChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin);
 

lib/icinga/checkable-notification.cpp

@@ -29,11 +29,11 @@
 using namespace icinga;
 
 boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const std::set<User::Ptr>&,
-    const NotificationType&, const CheckResult::Ptr&, const String&, const String&)> Checkable::OnNotificationSentToAllUsers;
+    const NotificationType&, const CheckResult::Ptr&, const String&, const String&,
-boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const std::set<User::Ptr>&,
+    const MessageOrigin::Ptr&)> Checkable::OnNotificationSentToAllUsers;
-    const NotificationType&, const CheckResult::Ptr&, const String&, const String&)> Checkable::OnNotificationSendStart;
 boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const User::Ptr&,
-    const NotificationType&, const CheckResult::Ptr&, const String&, const String&, const String&)> Checkable::OnNotificationSentToUser;
+    const NotificationType&, const CheckResult::Ptr&, const String&, const String&, const String&,
+    const MessageOrigin::Ptr&)> Checkable::OnNotificationSentToUser;
 
 void Checkable::ResetNotificationNumbers(void)
 {

lib/icinga/checkable.hpp

@@ -111,15 +111,12 @@ public:
 	static boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, std::set<Checkable::Ptr>, const MessageOrigin::Ptr&)> OnReachabilityChanged;
 	static boost::signals2::signal<void (const Checkable::Ptr&, NotificationType, const CheckResult::Ptr&,
 	    const String&, const String&, const MessageOrigin::Ptr&)> OnNotificationsRequested;
-	static boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const std::set<User::Ptr>&,
-	    const NotificationType&, const CheckResult::Ptr&, const String&,
-	    const String&)> OnNotificationSendStart;
 	static boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const User::Ptr&,
-	    const NotificationType&, const CheckResult::Ptr&, const String&,
+	    const NotificationType&, const CheckResult::Ptr&, const String&, const String&, const String&,
-	    const String&, const String&)> OnNotificationSentToUser;
+	    const MessageOrigin::Ptr&)> OnNotificationSentToUser;
 	static boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const std::set<User::Ptr>&,
 	    const NotificationType&, const CheckResult::Ptr&, const String&,
-	    const String&)> OnNotificationSentToAllUsers;
+	    const String&, const MessageOrigin::Ptr&)> OnNotificationSentToAllUsers;
 	static boost::signals2::signal<void (const Checkable::Ptr&, const String&, const String&, AcknowledgementType,
 					     bool, double, const MessageOrigin::Ptr&)> OnAcknowledgementSet;
 	static boost::signals2::signal<void (const Checkable::Ptr&, const MessageOrigin::Ptr&)> OnAcknowledgementCleared;

lib/icinga/clusterevents.cpp

@@ -49,6 +49,8 @@ REGISTER_APIFUNCTION(ClearAcknowledgement, event, &ClusterEvents::Acknowledgemen
 REGISTER_APIFUNCTION(UpdateRepository, event, &ClusterEvents::UpdateRepositoryAPIHandler);
 REGISTER_APIFUNCTION(ExecuteCommand, event, &ClusterEvents::ExecuteCommandAPIHandler);
 REGISTER_APIFUNCTION(SendNotifications, event, &ClusterEvents::SendNotificationsAPIHandler);
+REGISTER_APIFUNCTION(NotificationSentUser, event, &ClusterEvents::NotificationSentUserAPIHandler);
+REGISTER_APIFUNCTION(NotificationSentAllUsers, event, &ClusterEvents::NotificationSentAllUsersAPIHandler);
 
 static Timer::Ptr l_RepositoryTimer;
 
@@ -60,6 +62,8 @@ void ClusterEvents::StaticInitialize(void)
 	Checkable::OnForceNextCheckChanged.connect(&ClusterEvents::ForceNextCheckChangedHandler);
 	Checkable::OnForceNextNotificationChanged.connect(&ClusterEvents::ForceNextNotificationChangedHandler);
 	Checkable::OnNotificationsRequested.connect(&ClusterEvents::SendNotificationsHandler);
+	Checkable::OnNotificationSentToUser.connect(&ClusterEvents::NotificationSentUserHandler);
+	Checkable::OnNotificationSentToAllUsers.connect(&ClusterEvents::NotificationSentAllUsersHandler);
 
 	Checkable::OnAcknowledgementSet.connect(&ClusterEvents::AcknowledgementSetHandler);
 	Checkable::OnAcknowledgementCleared.connect(&ClusterEvents::AcknowledgementClearedHandler);
@@ -828,3 +832,221 @@ Value ClusterEvents::SendNotificationsAPIHandler(const MessageOrigin::Ptr& origi
 
 	return Empty;
 }
+
+void ClusterEvents::NotificationSentUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const User::Ptr& user,
+    NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const String& command,
+    const MessageOrigin::Ptr& origin)
+{
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+
+	if (!listener)
+		return;
+
+	Host::Ptr host;
+	Service::Ptr service;
+	tie(host, service) = GetHostService(checkable);
+
+	Dictionary::Ptr params = new Dictionary();
+	params->Set("host", host->GetName());
+	if (service)
+		params->Set("service", service->GetShortName());
+	params->Set("notification", notification->GetName());
+	params->Set("user", user->GetName());
+	params->Set("type", notificationType);
+	params->Set("cr", Serialize(cr));
+	params->Set("author", author);
+	params->Set("text", commentText);
+	params->Set("command", command);
+
+	Dictionary::Ptr message = new Dictionary();
+	message->Set("jsonrpc", "2.0");
+	message->Set("method", "event::NotificationSentUser");
+	message->Set("params", params);
+
+	listener->RelayMessage(origin, ConfigObject::Ptr(), message, true);
+}
+
+Value ClusterEvents::NotificationSentUserAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
+{
+	Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint();
+
+	if (!endpoint) {
+		Log(LogNotice, "ClusterEvents")
+		    << "Discarding 'sent notification to user' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed).";
+		return Empty;
+	}
+
+	if (!params)
+		return Empty;
+
+	Host::Ptr host = Host::GetByName(params->Get("host"));
+
+	if (!host)
+		return Empty;
+
+	Checkable::Ptr checkable;
+
+	if (params->Contains("service"))
+		checkable = host->GetServiceByShortName(params->Get("service"));
+	else
+		checkable = host;
+
+	if (!checkable)
+		return Empty;
+
+	if (origin->FromZone && origin->FromZone != Zone::GetLocalZone()) {
+		Log(LogNotice, "ClusterEvents")
+		    << "Discarding 'sent notification to user' message from '" << origin->FromClient->GetIdentity() << "': Unauthorized access.";
+		return Empty;
+	}
+
+	CheckResult::Ptr cr = new CheckResult();
+
+	Dictionary::Ptr vcr = params->Get("cr");
+	Array::Ptr vperf = vcr->Get("performance_data");
+	vcr->Remove("performance_data");
+
+	Deserialize(cr, params->Get("cr"), true);
+
+	NotificationType type = static_cast<NotificationType>(static_cast<int>(params->Get("type")));
+	String author = params->Get("author");
+	String text = params->Get("text");
+
+	Notification::Ptr notification = Notification::GetByName(params->Get("notification"));
+
+	if (!notification)
+		return Empty;
+
+	User::Ptr user = User::GetByName(params->Get("user"));
+
+	if (!user)
+		return Empty;
+
+	String command = params->Get("command");
+
+	Checkable::OnNotificationSentToUser(notification, checkable, user, type, cr, author, text, command, origin);
+
+	return Empty;
+}
+
+void ClusterEvents::NotificationSentAllUsersHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+    NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const MessageOrigin::Ptr& origin)
+{
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+
+	if (!listener)
+		return;
+
+	Host::Ptr host;
+	Service::Ptr service;
+	tie(host, service) = GetHostService(checkable);
+
+	Dictionary::Ptr params = new Dictionary();
+	params->Set("host", host->GetName());
+	if (service)
+		params->Set("service", service->GetShortName());
+	params->Set("notification", notification->GetName());
+
+	Array::Ptr ausers = new Array();
+	BOOST_FOREACH(const User::Ptr& user, users) {
+		ausers->Add(user->GetName());
+	}
+	params->Set("users", ausers);
+
+	params->Set("type", notificationType);
+	params->Set("cr", Serialize(cr));
+	params->Set("author", author);
+	params->Set("text", commentText);
+
+	params->Set("last_notification", notification->GetLastNotification());
+	params->Set("next_notifications", notification->GetNextNotification());
+	params->Set("notification_number", notification->GetNotificationNumber());
+	params->Set("last_problem_notification", notification->GetLastProblemNotification());
+
+	Dictionary::Ptr message = new Dictionary();
+	message->Set("jsonrpc", "2.0");
+	message->Set("method", "event::NotificationSentAllUsers");
+	message->Set("params", params);
+
+	listener->RelayMessage(origin, ConfigObject::Ptr(), message, true);
+}
+
+Value ClusterEvents::NotificationSentAllUsersAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
+{
+	Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint();
+
+	if (!endpoint) {
+		Log(LogNotice, "ClusterEvents")
+		    << "Discarding 'sent notification to all users' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed).";
+		return Empty;
+	}
+
+	if (!params)
+		return Empty;
+
+	Host::Ptr host = Host::GetByName(params->Get("host"));
+
+	if (!host)
+		return Empty;
+
+	Checkable::Ptr checkable;
+
+	if (params->Contains("service"))
+		checkable = host->GetServiceByShortName(params->Get("service"));
+	else
+		checkable = host;
+
+	if (!checkable)
+		return Empty;
+
+	if (origin->FromZone && origin->FromZone != Zone::GetLocalZone()) {
+		Log(LogNotice, "ClusterEvents")
+		    << "Discarding 'sent notification to all users' message from '" << origin->FromClient->GetIdentity() << "': Unauthorized access.";
+		return Empty;
+	}
+
+	CheckResult::Ptr cr = new CheckResult();
+
+	Dictionary::Ptr vcr = params->Get("cr");
+	Array::Ptr vperf = vcr->Get("performance_data");
+	vcr->Remove("performance_data");
+
+	Deserialize(cr, params->Get("cr"), true);
+
+	NotificationType type = static_cast<NotificationType>(static_cast<int>(params->Get("type")));
+	String author = params->Get("author");
+	String text = params->Get("text");
+
+	Notification::Ptr notification = Notification::GetByName(params->Get("notification"));
+
+	if (!notification)
+		return Empty;
+
+	Array::Ptr ausers = params->Get("users");
+
+	if (!ausers)
+		return Empty;
+
+	std::set<User::Ptr> users;
+
+	{
+		ObjectLock olock(ausers);
+		BOOST_FOREACH(const String& auser, ausers) {
+			User::Ptr user = User::GetByName(auser);
+
+			if (!user)
+				continue;
+
+			users.insert(user);
+		}
+	}
+
+	notification->SetLastNotification(params->Get("last_notification"));
+	notification->SetNextNotification(params->Get("next_notification"));
+	notification->SetNotificationNumber(params->Get("notification_number"));
+	notification->SetLastProblemNotification(params->Get("last_problem_notification"));
+
+	Checkable::OnNotificationSentToAllUsers(notification, checkable, users, type, cr, author, text, origin);
+
+	return Empty;
+}

lib/icinga/clusterevents.hpp

@@ -70,6 +70,14 @@ public:
 	static void SendNotificationsHandler(const Checkable::Ptr& checkable, NotificationType type,
 	    const CheckResult::Ptr& cr, const String& author, const String& text, const MessageOrigin::Ptr& origin);
 	static Value SendNotificationsAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
+
+	static void NotificationSentUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const User::Ptr& user,
+	    NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const String& command, const MessageOrigin::Ptr& origin);
+	static Value NotificationSentUserAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
+
+	static void NotificationSentAllUsersHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
+	    NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const MessageOrigin::Ptr& origin);
+	static Value NotificationSentAllUsersAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
 };
 
 }

lib/icinga/notification.cpp

@@ -345,7 +345,11 @@ void Notification::BeginExecuteNotification(NotificationType type, const CheckRe
 		std::copy(members.begin(), members.end(), std::inserter(allUsers, allUsers.begin()));
 	}
 
-	Service::OnNotificationSendStart(this, checkable, allUsers, type, cr, author, text);
+	{
+		ObjectLock olock(this);
+		UpdateNotificationNumber();
+		SetLastNotification(Utility::GetTime());
+	}
 
 	std::set<User::Ptr> allNotifiedUsers;
 	Array::Ptr notifiedUsers = GetNotifiedUsers();
@@ -392,7 +396,7 @@ void Notification::BeginExecuteNotification(NotificationType type, const CheckRe
 		notifiedUsers->Clear();
 
 	/* used in db_ido for notification history */
-	Service::OnNotificationSentToAllUsers(this, checkable, allNotifiedUsers, type, cr, author, text);
+	Service::OnNotificationSentToAllUsers(this, checkable, allNotifiedUsers, type, cr, author, text, MessageOrigin::Ptr());
 }
 
 bool Notification::CheckNotificationUserFilters(NotificationType type, const User::Ptr& user, bool force)
@@ -474,14 +478,8 @@ void Notification::ExecuteNotificationHelper(NotificationType type, const User::
 
 		command->Execute(this, user, cr, type, author, text);
 
-		{
-			ObjectLock olock(this);
-			UpdateNotificationNumber();
-			SetLastNotification(Utility::GetTime());
-		}
-
 		/* required by compatlogger */
-		Service::OnNotificationSentToUser(this, GetCheckable(), user, type, cr, author, text, command->GetName());
+		Service::OnNotificationSentToUser(this, GetCheckable(), user, type, cr, author, text, command->GetName(), MessageOrigin::Ptr());
 
 		Log(LogInformation, "Notification")
 		    << "Completed sending notification '" << GetName()

lib/icinga/notification.ti

@@ -92,7 +92,7 @@ class Notification : CustomVarObject < NotificationNameComposer
 
 	[state] double last_notification;
 	[state] double next_notification;
-	[state, set_protected] Value notification_number;
+	[state] int notification_number;
 	[state] double last_problem_notification;
 
 	[config, navigation] name(Endpoint) command_endpoint (CommandEndpointRaw) {