Merge pull request #9029 from Icinga/feature/icingadb-history-deterministic-ids

Icinga DB: make icinga:history:stream:*#event_id deterministic

lib/icingadb/icingadb-objects.cpp

@@ -127,7 +127,7 @@ void IcingaDB::ConfigStaticInitialize()
 void IcingaDB::UpdateAllConfigObjects()
 {
 	m_Rcon->Sync();
-	m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "1"}, Prio::Heartbeat);
+	m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "2"}, Prio::Heartbeat);
 
 	Log(LogInformation, "IcingaDB") << "Starting initial config/status dump";
 	double startTime = Utility::GetTime();
@@ -1522,6 +1522,9 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul
 	if (!checkable)
 		return;
 
+	if (!cr)
+		return;
+
 	Host::Ptr host;
 	Service::Ptr service;
 
@@ -1536,9 +1539,15 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul
 		hard_state = service ? Convert::ToLong(service->GetLastHardState()) : Convert::ToLong(host->GetLastHardState());
 	}
 
+	auto eventTime (cr->GetExecutionEnd());
+	auto eventTs (TimestampToMilliseconds(eventTime));
+
+	Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(object)));
+	rawId->Add(eventTs);
+
 	std::vector<String> xAdd ({
 		"XADD", "icinga:history:stream:state", "*",
-		"id", Utility::NewUniqueID(),
+		"id", HashValue(rawId),
 		"environment_id", m_EnvironmentId,
 		"host_id", GetObjectIdentifier(host),
 		"state_type", Convert::ToString(type),
@@ -1548,8 +1557,8 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul
 		"previous_soft_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeSoft)),
 		"previous_hard_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeHard)),
 		"max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()),
-		"event_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())),
+		"event_time", Convert::ToString(eventTs),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("state_change", object, eventTime),
 		"event_type", "state_change"
 	});
 
@@ -1595,7 +1604,7 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul
 
 void IcingaDB::SendSentNotification(
 	const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
-	NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text
+	NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime
 )
 {
 	if (!m_Rcon || !m_Rcon->IsConnected())
@@ -1611,7 +1620,13 @@ void IcingaDB::SendSentNotification(
 	}
 
 	auto usersAmount (users.size());
-	auto notificationHistoryId = Utility::NewUniqueID();
+	auto sendTs (TimestampToMilliseconds(sendTime));
+
+	Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(notification)));
+	rawId->Add(GetNotificationTypeByEnum(type));
+	rawId->Add(sendTs);
+
+	auto notificationHistoryId (HashValue(rawId));
 
 	std::vector<String> xAdd ({
 		"XADD", "icinga:history:stream:notification", "*",
@@ -1625,8 +1640,8 @@ void IcingaDB::SendSentNotification(
 		"author", Utility::ValidateUTF8(author),
 		"text", Utility::ValidateUTF8(finalText),
 		"users_notified", Convert::ToString(usersAmount),
-		"send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())),
+		"send_time", Convert::ToString(sendTs),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("notification", notification, sendTime, type),
 		"event_type", "notification"
 	});
 
@@ -1687,7 +1702,7 @@ void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime)
 		"scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())),
 		"has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()),
 		"trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("downtime_start", downtime),
 		"event_type", "downtime_start"
 	});
 
@@ -1774,7 +1789,7 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime)
 		"has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()),
 		"trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())),
 		"cancel_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("downtime_end", downtime),
 		"event_type", "downtime_end"
 	});
 
@@ -1851,7 +1866,7 @@ void IcingaDB::SendAddedComment(const Comment::Ptr& comment)
 		"entry_type", Convert::ToString(comment->GetEntryType()),
 		"is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()),
 		"is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("comment_add", comment),
 		"event_type", "comment_add"
 	});
 
@@ -1908,7 +1923,7 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment)
 		"entry_type", Convert::ToString(comment->GetEntryType()),
 		"is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()),
 		"is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)),
-		"event_id", Utility::NewUniqueID(),
+		"event_id", CalcEventID("comment_remove", comment),
 		"event_type", "comment_remove"
 	});
 
@@ -1969,8 +1984,7 @@ void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double change
 		"environment_id", m_EnvironmentId,
 		"host_id", GetObjectIdentifier(host),
 		"flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()),
-		"flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()),
+		"flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh())
-		"event_id", Utility::NewUniqueID()
 	});
 
 	if (service) {
@@ -2012,6 +2026,8 @@ void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double change
 
 	xAdd.emplace_back("start_time");
 	xAdd.emplace_back(Convert::ToString(startTime));
+	xAdd.emplace_back("event_id");
+	xAdd.emplace_back(CalcEventID(checkable->IsFlapping() ? "flapping_start" : "flapping_end", checkable, startTime));
 	xAdd.emplace_back("id");
 	xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), startTime})));
 
@@ -2056,7 +2072,6 @@ void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const Str
 
 	std::vector<String> xAdd ({
 		"XADD", "icinga:history:stream:acknowledgement", "*",
-		"event_id", Utility::NewUniqueID(),
 		"environment_id", m_EnvironmentId,
 		"host_id", GetObjectIdentifier(host),
 		"event_type", "ack_set",
@@ -2092,6 +2107,8 @@ void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const Str
 
 	xAdd.emplace_back("set_time");
 	xAdd.emplace_back(Convert::ToString(setTime));
+	xAdd.emplace_back("event_id");
+	xAdd.emplace_back(CalcEventID("ack_set", checkable, setTime));
 	xAdd.emplace_back("id");
 	xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), setTime})));
 
@@ -2109,7 +2126,6 @@ void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const
 
 	std::vector<String> xAdd ({
 		"XADD", "icinga:history:stream:acknowledgement", "*",
-		"event_id", Utility::NewUniqueID(),
 		"environment_id", m_EnvironmentId,
 		"host_id", GetObjectIdentifier(host),
 		"clear_time", Convert::ToString(TimestampToMilliseconds(changeTime)),
@@ -2137,6 +2153,8 @@ void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const
 
 	xAdd.emplace_back("set_time");
 	xAdd.emplace_back(Convert::ToString(setTime));
+	xAdd.emplace_back("event_id");
+	xAdd.emplace_back(CalcEventID("ack_clear", checkable, setTime));
 	xAdd.emplace_back("id");
 	xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), setTime})));
 
@@ -2374,10 +2392,11 @@ void IcingaDB::NotificationSentToAllUsersHandler(
 )
 {
 	auto rws (ConfigType::GetObjectsByType<IcingaDB>());
+	auto sendTime (notification->GetLastNotification());
 
 	if (!rws.empty()) {
 		for (auto& rw : rws) {
-			rw->SendSentNotification(notification, checkable, users, type, cr, author, text);
+			rw->SendSentNotification(notification, checkable, users, type, cr, author, text, sendTime);
 		}
 	}
 }

lib/icingadb/icingadb-utility.cpp

@@ -80,6 +80,28 @@ String IcingaDB::GetObjectIdentifier(const ConfigObject::Ptr& object)
 	return HashValue(new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(object))));
 }
 
+/**
+ * Calculates a deterministic history event ID like SHA1(env, eventType, x...[, nt][, eventTime])
+ *
+ * Where SHA1(env, x...) = GetObjectIdentifier(object)
+ */
+String IcingaDB::CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime, NotificationType nt)
+{
+	Array::Ptr rawId = new Array(GetObjectIdentifiersWithoutEnv(object));
+	rawId->Insert(0, GetEnvironment());
+	rawId->Insert(1, eventType);
+
+	if (nt) {
+		rawId->Add(GetNotificationTypeByEnum(nt));
+	}
+
+	if (eventTime) {
+		rawId->Add(TimestampToMilliseconds(eventTime));
+	}
+
+	return HashValue(std::move(rawId));
+}
+
 static const std::set<String> metadataWhitelist ({"package", "source_location", "templates"});
 
 /**
@@ -149,6 +171,32 @@ Dictionary::Ptr IcingaDB::SerializeVars(const CustomVarObject::Ptr& object)
 	return res;
 }
 
+const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type)
+{
+	switch (type) {
+		case NotificationDowntimeStart:
+			return "downtime_start";
+		case NotificationDowntimeEnd:
+			return "downtime_end";
+		case NotificationDowntimeRemoved:
+			return "downtime_removed";
+		case NotificationCustom:
+			return "custom";
+		case NotificationAcknowledgement:
+			return "acknowledgement";
+		case NotificationProblem:
+			return "problem";
+		case NotificationRecovery:
+			return "recovery";
+		case NotificationFlappingStart:
+			return "flapping_start";
+		case NotificationFlappingEnd:
+			return "flapping_end";
+	}
+
+	VERIFY(!"Invalid notification type.");
+}
+
 static const std::set<String> propertiesBlacklistEmpty;
 
 String IcingaDB::HashValue(const Value& value)

lib/icingadb/icingadb.hpp

@@ -81,7 +81,7 @@ private:
 
 	void SendSentNotification(
 		const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,
-		NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text
+		NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime
 	);
 
 	void SendStartedDowntime(const Downtime::Ptr& downtime);
@@ -107,8 +107,10 @@ private:
 
 	static ArrayData GetObjectIdentifiersWithoutEnv(const ConfigObject::Ptr& object);
 	static String GetObjectIdentifier(const ConfigObject::Ptr& object);
+	static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0));
 	static String GetEnvironment();
 	static Dictionary::Ptr SerializeVars(const CustomVarObject::Ptr& object);
+	static const char* GetNotificationTypeByEnum(NotificationType type);
 
 	static String HashValue(const Value& value);
 	static String HashValue(const Value& value, const std::set<String>& propertiesBlacklist, bool propertiesWhitelist = false);