From 4f6c87402e401c0523fa24756413c0cf485245e5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 17:45:12 +0200 Subject: [PATCH 1/4] Icinga DB: make icinga:history:stream:state#id deterministic ... i.e. UUID -> SHA1(x..., check time) given that SHA1(x...) = checkable id. Rationale: allow both masters to write the same state history concurrently (while not in split-brain), so that REPLACE INTO deduplicates the same events written twice. --- lib/icingadb/icingadb-objects.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index f1f6b5f09..e1c757869 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1522,6 +1522,9 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul if (!checkable) return; + if (!cr) + return; + Host::Ptr host; Service::Ptr service; @@ -1536,9 +1539,14 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul hard_state = service ? Convert::ToLong(service->GetLastHardState()) : Convert::ToLong(host->GetLastHardState()); } + auto eventTime (TimestampToMilliseconds(cr->GetExecutionEnd())); + + Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(object))); + rawId->Add(eventTime); + std::vector xAdd ({ "XADD", "icinga:history:stream:state", "*", - "id", Utility::NewUniqueID(), + "id", HashValue(rawId), "environment_id", m_EnvironmentId, "host_id", GetObjectIdentifier(host), "state_type", Convert::ToString(type), @@ -1548,7 +1556,7 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul "previous_soft_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeSoft)), "previous_hard_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeHard)), "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), - "event_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), + "event_time", Convert::ToString(eventTime), "event_id", Utility::NewUniqueID(), "event_type", "state_change" }); From 6a75d5121bc4aa25cd269e1a0d71762fa2c45fc2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Oct 2021 18:25:19 +0200 Subject: [PATCH 2/4] Icinga DB: make icinga:history:stream:notification#id deterministic ... i.e. UUID -> SHA1(x..., send time) given that SHA1(x...) = notification id. Rationale: allow both masters to write the same notification history concurrently (while not in split-brain), so that REPLACE INTO deduplicates the same events written twice. --- lib/icingadb/icingadb-objects.cpp | 14 ++++++++++---- lib/icingadb/icingadb-utility.cpp | 26 ++++++++++++++++++++++++++ lib/icingadb/icingadb.hpp | 3 ++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e1c757869..e7b0706a4 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1603,7 +1603,7 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul void IcingaDB::SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, - NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime ) { if (!m_Rcon || !m_Rcon->IsConnected()) @@ -1620,10 +1620,15 @@ void IcingaDB::SendSentNotification( auto usersAmount (users.size()); auto notificationHistoryId = Utility::NewUniqueID(); + auto sendTs (TimestampToMilliseconds(sendTime)); + + Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(notification))); + rawId->Add(GetNotificationTypeByEnum(type)); + rawId->Add(sendTs); std::vector xAdd ({ "XADD", "icinga:history:stream:notification", "*", - "id", notificationHistoryId, + "id", HashValue(rawId), "environment_id", m_EnvironmentId, "notification_id", GetObjectIdentifier(notification), "host_id", GetObjectIdentifier(host), @@ -1633,7 +1638,7 @@ void IcingaDB::SendSentNotification( "author", Utility::ValidateUTF8(author), "text", Utility::ValidateUTF8(finalText), "users_notified", Convert::ToString(usersAmount), - "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), + "send_time", Convert::ToString(sendTs), "event_id", Utility::NewUniqueID(), "event_type", "notification" }); @@ -2368,10 +2373,11 @@ void IcingaDB::NotificationSentToAllUsersHandler( ) { auto rws (ConfigType::GetObjectsByType()); + auto sendTime (notification->GetLastNotification()); if (!rws.empty()) { for (auto& rw : rws) { - rw->SendSentNotification(notification, checkable, users, type, cr, author, text); + rw->SendSentNotification(notification, checkable, users, type, cr, author, text, sendTime); } } } diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 8ca9e298e..74203ab9b 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -149,6 +149,32 @@ Dictionary::Ptr IcingaDB::SerializeVars(const CustomVarObject::Ptr& object) return res; } +const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type) +{ + switch (type) { + case NotificationDowntimeStart: + return "downtime_start"; + case NotificationDowntimeEnd: + return "downtime_end"; + case NotificationDowntimeRemoved: + return "downtime_removed"; + case NotificationCustom: + return "custom"; + case NotificationAcknowledgement: + return "acknowledgement"; + case NotificationProblem: + return "problem"; + case NotificationRecovery: + return "recovery"; + case NotificationFlappingStart: + return "flapping_start"; + case NotificationFlappingEnd: + return "flapping_end"; + } + + VERIFY(!"Invalid notification type."); +} + static const std::set propertiesBlacklistEmpty; String IcingaDB::HashValue(const Value& value) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index df49fd17d..90c91a845 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -81,7 +81,7 @@ private: void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, - NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text, double sendTime ); void SendStartedDowntime(const Downtime::Ptr& downtime); @@ -109,6 +109,7 @@ private: static String GetObjectIdentifier(const ConfigObject::Ptr& object); static String GetEnvironment(); static Dictionary::Ptr SerializeVars(const CustomVarObject::Ptr& object); + static const char* GetNotificationTypeByEnum(NotificationType type); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); From 1b06444b8d330a463100a813554dfb0770c9c205 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 11 Oct 2021 17:36:40 +0200 Subject: [PATCH 3/4] Icinga DB: make icinga:history:stream:*#event_id deterministic ... i.e. UUID -> SHA1(env, eventType, x...) given that SHA1(env, x...) = type-specific ID. Rationale: allow both masters to write the same history concurrently (while not in split-brain), so that REPLACE INTO deduplicates the same events written twice. * ack: SHA1(env, "ack_set"|"ack_clear", checkable.name, setTime) * comment: SHA1(env, "comment_add"|"comment_remove", comment.name) * downtime: SHA1(env, "downtime_start"|"downtime_end", downtime.name) * flapping: SHA1(env, "flapping_start"|"flapping_end", checkable.name, startTime) * notification: SHA1(env, "notification", notification.name, notificationType, sendTime) * state: SHA1(env, "state_change", checkable.name, changeTime) --- lib/icingadb/icingadb-objects.cpp | 35 ++++++++++++++++++------------- lib/icingadb/icingadb-utility.cpp | 22 +++++++++++++++++++ lib/icingadb/icingadb.hpp | 1 + 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e7b0706a4..ff3994bc9 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1539,10 +1539,11 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul hard_state = service ? Convert::ToLong(service->GetLastHardState()) : Convert::ToLong(host->GetLastHardState()); } - auto eventTime (TimestampToMilliseconds(cr->GetExecutionEnd())); + auto eventTime (cr->GetExecutionEnd()); + auto eventTs (TimestampToMilliseconds(eventTime)); Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(object))); - rawId->Add(eventTime); + rawId->Add(eventTs); std::vector xAdd ({ "XADD", "icinga:history:stream:state", "*", @@ -1556,8 +1557,8 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul "previous_soft_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeSoft)), "previous_hard_state", Convert::ToString(GetPreviousState(checkable, service, StateTypeHard)), "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), - "event_time", Convert::ToString(eventTime), - "event_id", Utility::NewUniqueID(), + "event_time", Convert::ToString(eventTs), + "event_id", CalcEventID("state_change", object, eventTime), "event_type", "state_change" }); @@ -1619,16 +1620,17 @@ void IcingaDB::SendSentNotification( } auto usersAmount (users.size()); - auto notificationHistoryId = Utility::NewUniqueID(); auto sendTs (TimestampToMilliseconds(sendTime)); Array::Ptr rawId = new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(notification))); rawId->Add(GetNotificationTypeByEnum(type)); rawId->Add(sendTs); + auto notificationHistoryId (HashValue(rawId)); + std::vector xAdd ({ "XADD", "icinga:history:stream:notification", "*", - "id", HashValue(rawId), + "id", notificationHistoryId, "environment_id", m_EnvironmentId, "notification_id", GetObjectIdentifier(notification), "host_id", GetObjectIdentifier(host), @@ -1639,7 +1641,7 @@ void IcingaDB::SendSentNotification( "text", Utility::ValidateUTF8(finalText), "users_notified", Convert::ToString(usersAmount), "send_time", Convert::ToString(sendTs), - "event_id", Utility::NewUniqueID(), + "event_id", CalcEventID("notification", notification, sendTime, type), "event_type", "notification" }); @@ -1700,7 +1702,7 @@ void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), - "event_id", Utility::NewUniqueID(), + "event_id", CalcEventID("downtime_start", downtime), "event_type", "downtime_start" }); @@ -1780,7 +1782,7 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), "cancel_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), - "event_id", Utility::NewUniqueID(), + "event_id", CalcEventID("downtime_end", downtime), "event_type", "downtime_end" }); @@ -1850,7 +1852,7 @@ void IcingaDB::SendAddedComment(const Comment::Ptr& comment) "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), "is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)), - "event_id", Utility::NewUniqueID(), + "event_id", CalcEventID("comment_add", comment), "event_type", "comment_add" }); @@ -1907,7 +1909,7 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), "is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)), - "event_id", Utility::NewUniqueID(), + "event_id", CalcEventID("comment_remove", comment), "event_type", "comment_remove" }); @@ -1968,8 +1970,7 @@ void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double change "environment_id", m_EnvironmentId, "host_id", GetObjectIdentifier(host), "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), - "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()), - "event_id", Utility::NewUniqueID() + "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()) }); if (service) { @@ -2011,6 +2012,8 @@ void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double change xAdd.emplace_back("start_time"); xAdd.emplace_back(Convert::ToString(startTime)); + xAdd.emplace_back("event_id"); + xAdd.emplace_back(CalcEventID(checkable->IsFlapping() ? "flapping_start" : "flapping_end", checkable, startTime)); xAdd.emplace_back("id"); xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), startTime}))); @@ -2055,7 +2058,6 @@ void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const Str std::vector xAdd ({ "XADD", "icinga:history:stream:acknowledgement", "*", - "event_id", Utility::NewUniqueID(), "environment_id", m_EnvironmentId, "host_id", GetObjectIdentifier(host), "event_type", "ack_set", @@ -2091,6 +2093,8 @@ void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const Str xAdd.emplace_back("set_time"); xAdd.emplace_back(Convert::ToString(setTime)); + xAdd.emplace_back("event_id"); + xAdd.emplace_back(CalcEventID("ack_set", checkable, setTime)); xAdd.emplace_back("id"); xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), setTime}))); @@ -2108,7 +2112,6 @@ void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const std::vector xAdd ({ "XADD", "icinga:history:stream:acknowledgement", "*", - "event_id", Utility::NewUniqueID(), "environment_id", m_EnvironmentId, "host_id", GetObjectIdentifier(host), "clear_time", Convert::ToString(TimestampToMilliseconds(changeTime)), @@ -2136,6 +2139,8 @@ void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const xAdd.emplace_back("set_time"); xAdd.emplace_back(Convert::ToString(setTime)); + xAdd.emplace_back("event_id"); + xAdd.emplace_back(CalcEventID("ack_clear", checkable, setTime)); xAdd.emplace_back("id"); xAdd.emplace_back(HashValue(new Array({GetEnvironment(), checkable->GetName(), setTime}))); diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 74203ab9b..3b1713cde 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -80,6 +80,28 @@ String IcingaDB::GetObjectIdentifier(const ConfigObject::Ptr& object) return HashValue(new Array(Prepend(GetEnvironment(), GetObjectIdentifiersWithoutEnv(object)))); } +/** + * Calculates a deterministic history event ID like SHA1(env, eventType, x...[, nt][, eventTime]) + * + * Where SHA1(env, x...) = GetObjectIdentifier(object) + */ +String IcingaDB::CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime, NotificationType nt) +{ + Array::Ptr rawId = new Array(GetObjectIdentifiersWithoutEnv(object)); + rawId->Insert(0, GetEnvironment()); + rawId->Insert(1, eventType); + + if (nt) { + rawId->Add(GetNotificationTypeByEnum(nt)); + } + + if (eventTime) { + rawId->Add(TimestampToMilliseconds(eventTime)); + } + + return HashValue(std::move(rawId)); +} + static const std::set metadataWhitelist ({"package", "source_location", "templates"}); /** diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 90c91a845..eef4267bf 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -107,6 +107,7 @@ private: static ArrayData GetObjectIdentifiersWithoutEnv(const ConfigObject::Ptr& object); static String GetObjectIdentifier(const ConfigObject::Ptr& object); + static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0)); static String GetEnvironment(); static Dictionary::Ptr SerializeVars(const CustomVarObject::Ptr& object); static const char* GetNotificationTypeByEnum(NotificationType type); From 9fa92aff97dac615fc6a9bef4358556f74489145 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 2 Nov 2021 15:00:55 +0100 Subject: [PATCH 4/4] Icinga DB: raise icinga:schema 1 -> 2 --- lib/icingadb/icingadb-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index ff3994bc9..daf515856 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -127,7 +127,7 @@ void IcingaDB::ConfigStaticInitialize() void IcingaDB::UpdateAllConfigObjects() { m_Rcon->Sync(); - m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "1"}, Prio::Heartbeat); + m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "2"}, Prio::Heartbeat); Log(LogInformation, "IcingaDB") << "Starting initial config/status dump"; double startTime = Utility::GetTime();