Introduce Notification#command_endpoint

for delegating notification execution to a specific endpoint.

doc/09-object-types.md

@@ -480,6 +480,7 @@ Configuration Attributes:
   user\_groups              | Array of object names | **Required.** A list of user group names who should be notified. **Optional.** if the `users` attribute is set.
   times                     | Dictionary            | **Optional.** A dictionary containing `begin` and `end` attributes for the notification. If `end` is set to 0, `Notifications` are disabled permanently. Please read the [notification delay](03-monitoring-basics.md#notification-delay) chapter for details.
   command                   | Object name           | **Required.** The name of the notification command which should be executed when the notification is triggered.
+  command\_endpoint         | Object name           | **Optional.** The endpoint where commands are executed on.
   interval                  | Duration              | **Optional.** The notification interval (in seconds). This interval is used for active notifications. Defaults to 30 minutes. If set to 0, [re-notifications](03-monitoring-basics.md#disable-renotification) are disabled.
   period                    | Object name           | **Optional.** The name of a time period which determines when this notification should be triggered. Not set by default (effectively 24x7).
   zone		            | Object name           | **Optional.** The zone this object is a member of. Please read the [distributed monitoring](06-distributed-monitoring.md#distributed-monitoring) chapter for details.

lib/icinga/clusterevents-check.cpp

@@ -316,7 +316,7 @@ void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, cons
 				throw;
 			}
 		}
-	} else if (command_type == "notification_command" && params->Contains("source")) {
+	} else if (command_type == "notification_command") {
 		/* Get user */
 		User::Ptr user = new User();
 		Dictionary::Ptr attrs = new Dictionary();
@@ -340,14 +340,18 @@ void ClusterEvents::ExecuteCheckFromQueue(const MessageOrigin::Ptr& origin, cons
 			NotificationCommand::Ptr notificationCommand = NotificationCommand::GetByName(command);
 
 			notificationCommand->Execute(notification, user, cr, NotificationType::NotificationCustom,
-				author, "");
+				author, "", macros, true);
 		} catch (const std::exception& ex) {
-			String output = "Exception occurred during notification '" + notification->GetName()
+			if (params->Contains("source")) {
-				+ "' for checkable '" + notification->GetCheckable()->GetName()
+				String output = "Exception occurred during notification '" + notification->GetName()
-				+ "' and user '" + user->GetName() + "' using command '" + command + "': "
+					+ "' for checkable '" + notification->GetCheckable()->GetName()
-				+ DiagnosticInformation(ex, false);
+					+ "' and user '" + user->GetName() + "' using command '" + command + "': "
-			double now = Utility::GetTime();
+					+ DiagnosticInformation(ex, false);
-			SendEventExecutedCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+				double now = Utility::GetTime();
+				SendEventExecutedCommand(params, ServiceUnknown, output, now, now, listener, origin, sourceEndpoint);
+			} else {
+				throw;
+			}
 		}
 	}
 }

lib/icinga/notification.cpp

@@ -123,6 +123,24 @@ void Notification::OnAllConfigLoaded()
 	if (!m_Checkable)
 		BOOST_THROW_EXCEPTION(ScriptError("Notification object refers to a host/service which doesn't exist.", GetDebugInfo()));
 
+	Endpoint::Ptr endpoint = GetCommandEndpoint();
+
+	if (endpoint) {
+		Zone::Ptr myZone = static_pointer_cast<Zone>(GetZone());
+
+		if (myZone) {
+			Zone::Ptr cmdZone = endpoint->GetZone();
+
+			if (cmdZone != myZone && cmdZone->GetParent() != myZone) {
+				BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
+					"Command endpoint must be in zone '" + myZone->GetName() + "' or in a direct child zone thereof."));
+			}
+		} else {
+			BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
+				"Notification with command endpoint requires a zone. Please check the troubleshooting documentation."));
+		}
+	}
+
 	GetCheckable()->RegisterNotification(this);
 }
 
@@ -604,7 +622,8 @@ void Notification::ExecuteNotificationHelper(NotificationType type, const User::
 {
 	String notificationName = GetName();
 	String userName = user->GetName();
-	String checkableName = GetCheckable()->GetName();
+	auto checkable (GetCheckable());
+	String checkableName = checkable->GetName();
 
 	NotificationCommand::Ptr command = GetCommand();
 
@@ -615,9 +634,53 @@ void Notification::ExecuteNotificationHelper(NotificationType type, const User::
 	}
 
 	String commandName = command->GetName();
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+	Endpoint::Ptr endpoint = GetCommandEndpoint();
+	bool local = !endpoint || endpoint == Endpoint::GetLocalEndpoint();
+
+	if (!local && !(endpoint->GetCapabilities() & (uint_fast64_t)ApiCapabilities::ExecuteNotificationCommand)) {
+		Log(LogWarning, "Notification")
+			<< "Sending notification '" << GetName() << "' locally as its command endpoint '"
+			<< endpoint->GetName() << "' doesn't support executing notification commands. Consider upgrading it.";
+		local = true;
+	}
+
+	if (!local && (!endpoint->GetConnected() || !listener)) {
+		Log(LogWarning, "Notification")
+			<< "Sending notification '" << GetName() << "' locally as its command endpoint '"
+			<< endpoint->GetName() << "' is not connected.";
+		local = true;
+	}
 
 	try {
-		command->Execute(this, user, cr, type, author, text);
+		if (local) {
+			command->Execute(this, user, cr, type, author, text);
+		} else {
+			Dictionary::Ptr macros = new Dictionary();
+			Host::Ptr host;
+			Service::Ptr service;
+			Dictionary::Ptr params = new Dictionary();
+			Dictionary::Ptr message = new Dictionary();
+
+			command->Execute(this, user, cr, type, author, text, macros, false);
+			tie(host, service) = GetHostService(checkable);
+
+			params->Set("command_type", "notification_command");
+			params->Set("command", command->GetName());
+			params->Set("macros", macros);
+			params->Set("notification", GetName());
+			params->Set("user", user->GetName());
+			params->Set("host", host->GetName());
+
+			if (service)
+				params->Set("service", service->GetShortName());
+
+			message->Set("jsonrpc", "2.0");
+			message->Set("method", "event::ExecuteCommand");
+			message->Set("params", params);
+
+			listener->SyncSendMessage(endpoint, message);
+		}
 
 		/* required by compatlogger */
 		Service::OnNotificationSentToUser(this, GetCheckable(), user, type, cr, author, text, commandName, nullptr);

lib/remote/apilistener.hpp

@@ -71,8 +71,9 @@ enum class ApiCapabilities : uint_fast64_t
 	ExecuteArbitraryCommand = 1u << 0u,
 	IfwApiCheckCommand = 1u << 1u,
 	HostChildrenInheritObjectAuthority = 1u << 2u,
+	ExecuteNotificationCommand = 1u << 3u,
 
-	MyCapabilities = ExecuteArbitraryCommand | IfwApiCheckCommand | HostChildrenInheritObjectAuthority
+	MyCapabilities = ExecuteArbitraryCommand | IfwApiCheckCommand | HostChildrenInheritObjectAuthority | ExecuteNotificationCommand
 };
 
 /**