Sync comment and downtime removal info for Icinga DB history

When a comment or downtime is removed manually, the name of the requestor and
timestamp have to be synced to other nodes in the cluster to allow all of them
to generate a consistent Icinga DB history stream.

refs #9101

doc/19-technical-concepts.md

@@ -1938,6 +1938,39 @@ Message updates will be dropped when:
 * Checkable does not exist.
 * Origin endpoint's zone is not allowed to access this checkable.
 
+#### event::SetRemovalInfo <a id="technical-concepts-json-rpc-messages-event-setremovalinfo"></a>
+
+> Location: `clusterevents.cpp`
+
+##### Message Body
+
+Key       | Value
+----------|---------
+jsonrpc   | 2.0
+method    | event::SetRemovalInfo
+params    | Dictionary
+
+##### Params
+
+Key            | Type        | Description
+---------------|-------------|---------------------------------
+object\_type   | String      | Object type (`"Comment"` or `"Downtime"`)
+object\_name   | String      | Object name
+removed\_by    | String      | Name of the removal requestor
+remove\_time   | Timestamp   | Time of the remove operation
+
+##### Functions
+
+**Event Sender**: `Comment::OnRemovalInfoChanged` and `Downtime::OnRemovalInfoChanged`
+**Event Receiver**: `SetRemovalInfoAPIHandler`
+
+This message is used to synchronize information about manual comment and downtime removals before deleting the
+corresponding object.
+
+##### Permissions
+
+This message is only accepted from the local zone and from parent zones.
+
 #### config::Update <a id="technical-concepts-json-rpc-messages-config-update"></a>
 
 > Location: `apilistener-filesync.cpp`

lib/icinga/apiactions.cpp

@@ -311,12 +311,7 @@ Dictionary::Ptr ApiActions::RemoveComment(const ConfigObject::Ptr& object,
 		std::set<Comment::Ptr> comments = checkable->GetComments();
 
 		for (const Comment::Ptr& comment : comments) {
-			{
+			Comment::RemoveComment(comment->GetName(), true, author);
-				ObjectLock oLock (comment);
-				comment->SetRemovedBy(author);
-			}
-
-			Comment::RemoveComment(comment->GetName());
 		}
 
 		return ApiActions::CreateResult(200, "Successfully removed all comments for object '" + checkable->GetName() + "'.");
@@ -327,14 +322,9 @@ Dictionary::Ptr ApiActions::RemoveComment(const ConfigObject::Ptr& object,
 	if (!comment)
 		return ApiActions::CreateResult(404, "Cannot remove non-existent comment object.");
 
-	{
-		ObjectLock oLock (comment);
-		comment->SetRemovedBy(author);
-	}
-
 	String commentName = comment->GetName();
 
-	Comment::RemoveComment(commentName);
+	Comment::RemoveComment(commentName, true, author);
 
 	return ApiActions::CreateResult(200, "Successfully removed comment '" + commentName + "'.");
 }
@@ -507,15 +497,10 @@ Dictionary::Ptr ApiActions::RemoveDowntime(const ConfigObject::Ptr& object,
 		std::set<Downtime::Ptr> downtimes = checkable->GetDowntimes();
 
 		for (const Downtime::Ptr& downtime : downtimes) {
-			{
-				ObjectLock oLock (downtime);
-				downtime->SetRemovedBy(author);
-			}
-
 			childCount += downtime->GetChildren().size();
 
 			try {
-				Downtime::RemoveDowntime(downtime->GetName(), true, true);
+				Downtime::RemoveDowntime(downtime->GetName(), true, true, false, author);
 			} catch (const invalid_downtime_removal_error& error) {
 				Log(LogWarning, "ApiActions") << error.what();
 
@@ -532,16 +517,11 @@ Dictionary::Ptr ApiActions::RemoveDowntime(const ConfigObject::Ptr& object,
 	if (!downtime)
 		return ApiActions::CreateResult(404, "Cannot remove non-existent downtime object.");
 
-	{
-		ObjectLock oLock (downtime);
-		downtime->SetRemovedBy(author);
-	}
-
 	childCount += downtime->GetChildren().size();
 
 	try {
 		String downtimeName = downtime->GetName();
-		Downtime::RemoveDowntime(downtimeName, true, true);
+		Downtime::RemoveDowntime(downtimeName, true, true, false, author);
 
 		return ApiActions::CreateResult(200, "Successfully removed downtime '" + downtimeName +
 			"' and " + std::to_string(childCount) + " child downtimes.");

lib/icinga/clusterevents.cpp

@@ -38,6 +38,7 @@ REGISTER_APIFUNCTION(NotificationSentUser, event, &ClusterEvents::NotificationSe
 REGISTER_APIFUNCTION(NotificationSentToAllUsers, event, &ClusterEvents::NotificationSentToAllUsersAPIHandler);
 REGISTER_APIFUNCTION(ExecutedCommand, event, &ClusterEvents::ExecutedCommandAPIHandler);
 REGISTER_APIFUNCTION(UpdateExecutions, event, &ClusterEvents::UpdateExecutionsAPIHandler);
+REGISTER_APIFUNCTION(SetRemovalInfo, event, &ClusterEvents::SetRemovalInfoAPIHandler);
 
 void ClusterEvents::StaticInitialize()
 {
@@ -55,6 +56,9 @@ void ClusterEvents::StaticInitialize()
 
 	Checkable::OnAcknowledgementSet.connect(&ClusterEvents::AcknowledgementSetHandler);
 	Checkable::OnAcknowledgementCleared.connect(&ClusterEvents::AcknowledgementClearedHandler);
+
+	Comment::OnRemovalInfoChanged.connect(&ClusterEvents::SetRemovalInfoHandler);
+	Downtime::OnRemovalInfoChanged.connect(&ClusterEvents::SetRemovalInfoHandler);
 }
 
 Dictionary::Ptr ClusterEvents::MakeCheckResultMessage(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
@@ -1307,3 +1311,62 @@ Value ClusterEvents::UpdateExecutionsAPIHandler(const MessageOrigin::Ptr& origin
 
 	return Empty;
 }
+
+void ClusterEvents::SetRemovalInfoHandler(const ConfigObject::Ptr& obj, const String& removedBy, double removeTime,
+	const MessageOrigin::Ptr& origin)
+{
+	ApiListener::Ptr listener = ApiListener::GetInstance();
+
+	if (!listener)
+		return;
+
+	Dictionary::Ptr params = new Dictionary();
+	params->Set("object_type", obj->GetReflectionType()->GetName());
+	params->Set("object_name", obj->GetName());
+	params->Set("removed_by", removedBy);
+	params->Set("remove_time", removeTime);
+
+	Dictionary::Ptr message = new Dictionary();
+	message->Set("jsonrpc", "2.0");
+	message->Set("method", "event::SetRemovalInfo");
+	message->Set("params", params);
+
+	listener->RelayMessage(origin, obj, message, true);
+}
+
+Value ClusterEvents::SetRemovalInfoAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
+{
+	Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint();
+
+	if (!endpoint || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'set removal info' message from '" << origin->FromClient->GetIdentity()
+			<< "': Invalid endpoint origin (client not allowed).";
+		return Empty;
+	}
+
+	String objectType = params->Get("object_type");
+	String objectName = params->Get("object_name");
+	String removedBy = params->Get("removed_by");
+	double removeTime = params->Get("remove_time");
+
+	if (objectType == Comment::GetTypeName()) {
+		Comment::Ptr comment = Comment::GetByName(objectName);
+
+		if (comment) {
+			comment->SetRemovalInfo(removedBy, removeTime, origin);
+		}
+	} else if (objectType == Downtime::GetTypeName()) {
+		Downtime::Ptr downtime = Downtime::GetByName(objectName);
+
+		if (downtime) {
+			downtime->SetRemovalInfo(removedBy, removeTime, origin);
+		}
+	} else {
+		Log(LogNotice, "ClusterEvents")
+			<< "Discarding 'set removal info' message from '" << origin->FromClient->GetIdentity()
+			<< "': Unknown object type.";
+	}
+
+	return Empty;
+}

lib/icinga/clusterevents.hpp

@@ -69,6 +69,9 @@ public:
 	static Value ExecutedCommandAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
 	static Value UpdateExecutionsAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
 
+	static void SetRemovalInfoHandler(const ConfigObject::Ptr& obj, const String& removedBy, double removeTime, const MessageOrigin::Ptr& origin);
+	static Value SetRemovalInfoAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
+
 	static int GetCheckRequestQueueSize();
 	static void LogRemoteCheckQueueInformation();
 

lib/icinga/comment.cpp

@@ -18,6 +18,7 @@ static Timer::Ptr l_CommentsExpireTimer;
 
 boost::signals2::signal<void (const Comment::Ptr&)> Comment::OnCommentAdded;
 boost::signals2::signal<void (const Comment::Ptr&)> Comment::OnCommentRemoved;
+boost::signals2::signal<void (const Comment::Ptr&, const String&, double, const MessageOrigin::Ptr&)> Comment::OnRemovalInfoChanged;
 
 REGISTER_TYPE(Comment);
 
@@ -185,7 +186,8 @@ String Comment::AddComment(const Checkable::Ptr& checkable, CommentType entryTyp
 	return fullName;
 }
 
-void Comment::RemoveComment(const String& id, const MessageOrigin::Ptr& origin)
+void Comment::RemoveComment(const String& id, bool removedManually, const String& removedBy,
+	const MessageOrigin::Ptr& origin)
 {
 	Comment::Ptr comment = Comment::GetByName(id);
 
@@ -195,6 +197,10 @@ void Comment::RemoveComment(const String& id, const MessageOrigin::Ptr& origin)
 	Log(LogNotice, "Comment")
 		<< "Removed comment '" << comment->GetName() << "' from object '" << comment->GetCheckable()->GetName() << "'.";
 
+	if (removedManually) {
+		comment->SetRemovalInfo(removedBy, Utility::GetTime());
+	}
+
 	Array::Ptr errors = new Array();
 
 	if (!ConfigObjectUtility::DeleteObject(comment, false, errors, nullptr)) {
@@ -207,6 +213,17 @@ void Comment::RemoveComment(const String& id, const MessageOrigin::Ptr& origin)
 	}
 }
 
+void Comment::SetRemovalInfo(const String& removedBy, double removeTime, const MessageOrigin::Ptr& origin) {
+	{
+		ObjectLock olock(this);
+
+		SetRemovedBy(removedBy, false, origin);
+		SetRemoveTime(removeTime, false, origin);
+	}
+
+	OnRemovalInfoChanged(this, removedBy, removeTime, origin);
+}
+
 String Comment::GetCommentIDFromLegacyID(int id)
 {
 	std::unique_lock<std::mutex> lock(l_CommentMutex);

lib/icinga/comment.hpp

@@ -24,18 +24,22 @@ public:
 
 	static boost::signals2::signal<void (const Comment::Ptr&)> OnCommentAdded;
 	static boost::signals2::signal<void (const Comment::Ptr&)> OnCommentRemoved;
+	static boost::signals2::signal<void (const Comment::Ptr&, const String&, double, const MessageOrigin::Ptr&)> OnRemovalInfoChanged;
 
 	intrusive_ptr<Checkable> GetCheckable() const;
 
 	bool IsExpired() const;
 
+	void SetRemovalInfo(const String& removedBy, double removeTime, const MessageOrigin::Ptr& origin = nullptr);
+
 	static int GetNextCommentID();
 
 	static String AddComment(const intrusive_ptr<Checkable>& checkable, CommentType entryType,
 		const String& author, const String& text, bool persistent, double expireTime,
 		const String& id = String(), const MessageOrigin::Ptr& origin = nullptr);
 
-	static void RemoveComment(const String& id, const MessageOrigin::Ptr& origin = nullptr);
+	static void RemoveComment(const String& id, bool removedManually = false, const String& removedBy = "",
+		const MessageOrigin::Ptr& origin = nullptr);
 
 	static String GetCommentIDFromLegacyID(int id);
 

lib/icinga/comment.ti

@@ -73,6 +73,7 @@ class Comment : ConfigObject < CommentNameComposer
 	[state] int legacy_id;
 
 	[no_user_view, no_user_modify] String removed_by;
+	[no_user_view, no_user_modify] Timestamp remove_time;
 };
 
 }

lib/icinga/downtime.cpp

@@ -23,6 +23,7 @@ boost::signals2::signal<void (const Downtime::Ptr&)> Downtime::OnDowntimeAdded;
 boost::signals2::signal<void (const Downtime::Ptr&)> Downtime::OnDowntimeRemoved;
 boost::signals2::signal<void (const Downtime::Ptr&)> Downtime::OnDowntimeStarted;
 boost::signals2::signal<void (const Downtime::Ptr&)> Downtime::OnDowntimeTriggered;
+boost::signals2::signal<void (const Downtime::Ptr&, const String&, double, const MessageOrigin::Ptr&)> Downtime::OnRemovalInfoChanged;
 
 REGISTER_TYPE(Downtime);
 
@@ -331,7 +332,8 @@ Downtime::Ptr Downtime::AddDowntime(const Checkable::Ptr& checkable, const Strin
 	return downtime;
 }
 
-void Downtime::RemoveDowntime(const String& id, bool includeChildren, bool cancelled, bool expired, const MessageOrigin::Ptr& origin)
+void Downtime::RemoveDowntime(const String& id, bool includeChildren, bool cancelled, bool expired,
+	const String& removedBy, const MessageOrigin::Ptr& origin)
 {
 	Downtime::Ptr downtime = Downtime::GetByName(id);
 
@@ -351,7 +353,9 @@ void Downtime::RemoveDowntime(const String& id, bool includeChildren, bool cance
 		}
 	}
 
-	downtime->SetWasCancelled(cancelled);
+	if (cancelled) {
+		downtime->SetRemovalInfo(removedBy, Utility::GetTime());
+	}
 
 	Array::Ptr errors = new Array();
 
@@ -454,6 +458,17 @@ void Downtime::TriggerDowntime(double triggerTime)
 	OnDowntimeTriggered(this);
 }
 
+void Downtime::SetRemovalInfo(const String& removedBy, double removeTime, const MessageOrigin::Ptr& origin) {
+	{
+		ObjectLock olock(this);
+
+		SetRemovedBy(removedBy, false, origin);
+		SetRemoveTime(removeTime, false, origin);
+	}
+
+	OnRemovalInfoChanged(this, removedBy, removeTime, origin);
+}
+
 String Downtime::GetDowntimeIDFromLegacyID(int id)
 {
 	std::unique_lock<std::mutex> lock(l_DowntimeMutex);

lib/icinga/downtime.hpp

@@ -33,6 +33,7 @@ public:
 	static boost::signals2::signal<void (const Downtime::Ptr&)> OnDowntimeRemoved;
 	static boost::signals2::signal<void (const Downtime::Ptr&)> OnDowntimeStarted;
 	static boost::signals2::signal<void (const Downtime::Ptr&)> OnDowntimeTriggered;
+	static boost::signals2::signal<void (const Downtime::Ptr&, const String&, double, const MessageOrigin::Ptr&)> OnRemovalInfoChanged;
 
 	intrusive_ptr<Checkable> GetCheckable() const;
 
@@ -51,13 +52,15 @@ public:
 		const String& scheduledBy = String(), const String& parent = String(), const String& id = String(),
 		const MessageOrigin::Ptr& origin = nullptr);
 
-	static void RemoveDowntime(const String& id, bool includeChildren, bool cancelled, bool expired = false, const MessageOrigin::Ptr& origin = nullptr);
+	static void RemoveDowntime(const String& id, bool includeChildren, bool cancelled, bool expired = false,
+		const String& removedBy = "", const MessageOrigin::Ptr& origin = nullptr);
 
 	void RegisterChild(const Downtime::Ptr& downtime);
 	void UnregisterChild(const Downtime::Ptr& downtime);
 	std::set<Downtime::Ptr> GetChildren() const;
 
 	void TriggerDowntime(double triggerTime);
+	void SetRemovalInfo(const String& removedBy, double removeTime, const MessageOrigin::Ptr& origin = nullptr);
 
 	static String GetDowntimeIDFromLegacyID(int id);
 

lib/icinga/downtime.ti

@@ -68,7 +68,10 @@ class Downtime : ConfigObject < DowntimeNameComposer
 		default {{{ return new Array(); }}}
 	};
 	[state] int legacy_id;
-	[state] bool was_cancelled;
+	[state] Timestamp remove_time;
+	[no_storage] bool was_cancelled {
+		get {{{ return GetRemoveTime() > 0; }}}
+	};
 	[config] String config_owner;
 	[config] String config_owner_hash;
 	[config] String authoritative_zone;

lib/icingadb/icingadb-objects.cpp

@@ -1884,7 +1884,7 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime)
 		"scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())),
 		"has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()),
 		"trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())),
-		"cancel_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())),
+		"cancel_time", Convert::ToString(TimestampToMilliseconds(downtime->GetRemoveTime())),
 		"event_id", CalcEventID("downtime_end", downtime),
 		"event_type", "downtime_end"
 	});
@@ -2040,9 +2040,10 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment)
 		xAdd.emplace_back(GetObjectIdentifier(endpoint));
 	}
 
-	if (comment->GetExpireTime() < Utility::GetTime()) {
+	double removeTime = comment->GetRemoveTime();
+	if (removeTime > 0) {
 		xAdd.emplace_back("remove_time");
-		xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(Utility::GetTime())));
+		xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(removeTime)));
 		xAdd.emplace_back("has_been_removed");
 		xAdd.emplace_back("1");
 		xAdd.emplace_back("removed_by");