From a6d6cb788e436d84982d8a50ad53aba8ae4d2604 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 15:16:26 +0100 Subject: [PATCH 1/6] Icinga DB: Merge SendStatusUpdate into UpdateState Previously, both funktions did related operations but had unclear and confusing naming: - UpdateState updated the icinga:{host,service}:state Redis keys. - SendStatusUpdate sent a runtime update for the icinga:{host,service}:state. This commit merges both functions into one with a new mode parameter. The following modes are now supported: - Volatile: Update the icinga:{host,service}:state Redis key. - Full: Perform the volatile state update and in addition send a corresponding runtime update so that this state update gets written through to the persistent database by a running icingadb process. - RuntimeOnly: Special mode for callers that can ensure that a volatile update for the current state was already performed but has to be upgraded to a full update. refs #9063 --- lib/icingadb/icingadb-objects.cpp | 86 ++++++++++++++++++------------- lib/icingadb/icingadb.hpp | 10 +++- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 0d2db2120..844cff98a 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1091,7 +1091,23 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } -void IcingaDB::UpdateState(const Checkable::Ptr& checkable) +/** + * Update the state information of a checkable in Redis. + * + * What is updated exactly depends on the mode parameter: + * - Volatile: Update the volatile state information stored in icinga:host:state or icinga:service:state as well as + * the corresponding checksum stored in icinga:checksum:host:state or icinga:checksum:service:state. + * - RuntimeOnly: Write a runtime update to the icinga:runtime:state stream. It is up to the caller to ensure that + * identical volatile state information was already written before to avoid inconsistencies. This mode is only + * useful to upgrade a previous Volatile to a Full operation, otherwise Full should be used. + * - Full: Perform an update of all state information in Redis, that is updating the volatile information and sending + * a corresponding runtime update so that this state update gets written through to the persistent database by a + * running icingadb process. + * + * @param checkable State of this checkable is updated in Redis + * @param mode Mode of operation (StateUpdate::Volatile, StateUpdate::RuntimeOnly, or StateUpdate::Full) + */ +void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1101,9 +1117,34 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable) Dictionary::Ptr stateAttrs = SerializeState(checkable); - m_Rcon->FireAndForgetQuery({"HSET", m_PrefixConfigObject + objectType + ":state", objectKey, JsonEncode(stateAttrs)}, Prio::RuntimeStateSync); - m_Rcon->FireAndForgetQuery({"HSET", m_PrefixConfigCheckSum + objectType + ":state", objectKey, JsonEncode(new Dictionary({{"checksum", HashValue(stateAttrs)}}))}, Prio::RuntimeStateSync); + String redisStateKey = m_PrefixConfigObject + objectType + ":state"; + String redisChecksumKey = m_PrefixConfigCheckSum + objectType + ":state"; + String checksum = HashValue(stateAttrs); + if (mode & StateUpdate::Volatile) { + m_Rcon->FireAndForgetQueries({ + {"HSET", redisStateKey, objectKey, JsonEncode(stateAttrs)}, + {"HSET", redisChecksumKey, objectKey, JsonEncode(new Dictionary({{"checksum", checksum}}))}, + }, Prio::RuntimeStateSync); + } + + if (mode & StateUpdate::RuntimeOnly) { + ObjectLock olock(stateAttrs); + + std::vector streamadd({ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", + "runtime_type", "upsert", + "redis_key", redisStateKey, + "checksum", checksum, + }); + + for (const Dictionary::Pair& kv : stateAttrs) { + streamadd.emplace_back(kv.first); + streamadd.emplace_back(IcingaToStreamValue(kv.second)); + } + + m_Rcon->FireAndForgetQuery(std::move(streamadd), Prio::RuntimeStateStream); + } } // Used to update a single object, used for runtime updates @@ -1128,7 +1169,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd m_Rcon->FireAndForgetQuery({"HSET", m_PrefixConfigCheckSum + typeName + ":state", objectKey, JsonEncode(new Dictionary({{"checksum", checksum}}))}, Prio::RuntimeStateSync); if (runtimeUpdate) { - SendStatusUpdate(checkable); + UpdateState(checkable, StateUpdate::RuntimeOnly); } } @@ -1593,31 +1634,6 @@ unsigned short GetPreviousState(const Checkable::Ptr& checkable, const Service:: } } -void IcingaDB::SendStatusUpdate(const Checkable::Ptr& checkable) -{ - if (!m_Rcon || !m_Rcon->IsConnected()) - return; - - Host::Ptr host; - Service::Ptr service; - Dictionary::Ptr objectAttrs = SerializeState(checkable); - std::vector streamadd({"XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*"}); - ObjectLock olock(objectAttrs); - - tie(host, service) = GetHostService(checkable); - - objectAttrs->Set("checksum", HashValue(objectAttrs)); - objectAttrs->Set("redis_key", service ? "icinga:service:state" : "icinga:host:state"); - objectAttrs->Set("runtime_type", "upsert"); - - for (const Dictionary::Pair& kv : objectAttrs) { - streamadd.emplace_back(kv.first); - streamadd.emplace_back(IcingaToStreamValue(kv.second)); - } - - m_Rcon->FireAndForgetQuery(std::move(streamadd), Prio::RuntimeStateStream); -} - void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { if (!m_Rcon || !m_Rcon->IsConnected()) @@ -1635,7 +1651,7 @@ void IcingaDB::SendStateChange(const ConfigObject::Ptr& object, const CheckResul tie(host, service) = GetHostService(checkable); - SendStatusUpdate(checkable); + UpdateState(checkable, StateUpdate::RuntimeOnly); int hard_state; if (!cr) { @@ -2002,8 +2018,7 @@ void IcingaDB::SendAddedComment(const Comment::Ptr& comment) } m_Rcon->FireAndForgetQuery(std::move(xAdd), Prio::History); - UpdateState(checkable); - SendStatusUpdate(checkable); + UpdateState(checkable, StateUpdate::Full); } void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) @@ -2072,8 +2087,7 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) } m_Rcon->FireAndForgetQuery(std::move(xAdd), Prio::History); - UpdateState(checkable); - SendStatusUpdate(checkable); + UpdateState(checkable, StateUpdate::Full); } void IcingaDB::SendFlappingChange(const Checkable::Ptr& checkable, double changeTime, double flappingLastChange) @@ -2678,7 +2692,7 @@ void IcingaDB::FlappingChangeHandler(const Checkable::Ptr& checkable, double cha void IcingaDB::NewCheckResultHandler(const Checkable::Ptr& checkable) { for (auto& rw : ConfigType::GetObjectsByType()) { - rw->UpdateState(checkable); + rw->UpdateState(checkable, StateUpdate::Volatile); rw->SendNextUpdate(checkable); } } @@ -2686,7 +2700,7 @@ void IcingaDB::NewCheckResultHandler(const Checkable::Ptr& checkable) void IcingaDB::NextCheckChangedHandler(const Checkable::Ptr& checkable) { for (auto& rw : ConfigType::GetObjectsByType()) { - rw->UpdateState(checkable); + rw->UpdateState(checkable, StateUpdate::Volatile); rw->SendNextUpdate(checkable); } } diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 7a392a2ad..f0b62047f 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -57,6 +57,13 @@ private: std::mutex m_Mutex; }; + enum StateUpdate + { + Volatile = 1ull << 0, + RuntimeOnly = 1ull << 1, + Full = Volatile | RuntimeOnly, + }; + void OnConnectedHandler(); void PublishStatsTimerHandler(); @@ -70,12 +77,11 @@ private: std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); - void UpdateState(const Checkable::Ptr& checkable); + void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); - void SendStatusUpdate(const Checkable::Ptr& checkable); void SendStateChange(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); From 447884be72e95b578f7dacd53cd6d4d0678e9893 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 15:29:51 +0100 Subject: [PATCH 2/6] Icinga DB: don't reimplement volatile state update in SendConfigUpdate Sending a volatile state update is already implemented in UpdateState, so just use that function instead of generating the update queries. --- lib/icingadb/icingadb-objects.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 844cff98a..df8d3553a 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1161,16 +1161,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd CreateConfigUpdate(object, typeName, hMSets, runtimeUpdates, runtimeUpdate); Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { - String objectKey = GetObjectIdentifier(object); - Dictionary::Ptr state = SerializeState(checkable); - String checksum = HashValue(state); - - m_Rcon->FireAndForgetQuery({"HSET", m_PrefixConfigObject + typeName + ":state", objectKey, JsonEncode(state)}, Prio::RuntimeStateSync); - m_Rcon->FireAndForgetQuery({"HSET", m_PrefixConfigCheckSum + typeName + ":state", objectKey, JsonEncode(new Dictionary({{"checksum", checksum}}))}, Prio::RuntimeStateSync); - - if (runtimeUpdate) { - UpdateState(checkable, StateUpdate::RuntimeOnly); - } + UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile); } std::vector > transaction = {{"MULTI"}}; From f63268b0dd0ce28e046afb1b72b084295babc408 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 15:33:15 +0100 Subject: [PATCH 3/6] Icinga DB: make downtime events update the state tables but not write state history StateChangeHandler() is the function used when the actual hard/soft state changes and thus also writes state history. This is not desired in this case, instead, a runtime update should be generated, therefore call UpdateState() instead. refs #9063 --- lib/icingadb/icingadb-objects.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index df8d3553a..b43e749dc 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1800,6 +1800,9 @@ void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) Service::Ptr service; tie(host, service) = GetHostService(checkable); + /* Update checkable state as in_downtime may have changed. */ + UpdateState(checkable, StateUpdate::Full); + std::vector xAdd ({ "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), @@ -1885,6 +1888,9 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) if (downtime->GetTriggerTime() == 0) return; + /* Update checkable state as in_downtime may have changed. */ + UpdateState(checkable, StateUpdate::Full); + std::vector xAdd ({ "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), @@ -2626,8 +2632,6 @@ void IcingaDB::VersionChangedHandler(const ConfigObject::Ptr& object) void IcingaDB::DowntimeStartedHandler(const Downtime::Ptr& downtime) { - StateChangeHandler(downtime->GetCheckable()); - for (auto& rw : ConfigType::GetObjectsByType()) { rw->SendStartedDowntime(downtime); } @@ -2635,8 +2639,6 @@ void IcingaDB::DowntimeStartedHandler(const Downtime::Ptr& downtime) void IcingaDB::DowntimeRemovedHandler(const Downtime::Ptr& downtime) { - StateChangeHandler(downtime->GetCheckable()); - for (auto& rw : ConfigType::GetObjectsByType()) { rw->SendRemovedDowntime(downtime); } From 855e342b634feda6b67b1b84a9308394beb07daa Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 16:21:57 +0100 Subject: [PATCH 4/6] Icinga DB: make acknowledgement events update the state tables but not write state history StateChangeHandler() is the function used when the actual hard/soft state changes and thus also writes state history. This is not desired in this case, instead, a runtime update should be generated, therefore call UpdateState() instead. refs #9063 --- lib/icingadb/icingadb-objects.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index b43e749dc..fbfee0b64 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -84,14 +84,6 @@ void IcingaDB::ConfigStaticInitialize() AcknowledgementClearedHandler(checkable, removedBy, changeTime); }); - Checkable::OnAcknowledgementSet.connect([](const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool, bool persistent, double changeTime, double expiry, const MessageOrigin::Ptr&) { - IcingaDB::StateChangeHandler(checkable); - }); - /* triggered when acknowledged host/service goes back to ok and when the acknowledgement gets deleted */ - Checkable::OnAcknowledgementCleared.connect([](const Checkable::Ptr& checkable, const String&, double, const MessageOrigin::Ptr&) { - IcingaDB::StateChangeHandler(checkable); - }); - /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect([](const ConfigObject::Ptr& object, const Value&) { IcingaDB::VersionChangedHandler(object); @@ -2187,6 +2179,9 @@ void IcingaDB::SendAcknowledgementSet(const Checkable::Ptr& checkable, const Str Service::Ptr service; tie(host, service) = GetHostService(checkable); + /* Update checkable state as is_acknowledged may have changed. */ + UpdateState(checkable, StateUpdate::Full); + std::vector xAdd ({ "XADD", "icinga:history:stream:acknowledgement", "*", "environment_id", m_EnvironmentId, @@ -2241,6 +2236,9 @@ void IcingaDB::SendAcknowledgementCleared(const Checkable::Ptr& checkable, const Service::Ptr service; tie(host, service) = GetHostService(checkable); + /* Update checkable state as is_acknowledged may have changed. */ + UpdateState(checkable, StateUpdate::Full); + std::vector xAdd ({ "XADD", "icinga:history:stream:acknowledgement", "*", "environment_id", m_EnvironmentId, From cf73c6136b78c94ecfe19b17b8d9312e1891c4bd Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 16:39:15 +0100 Subject: [PATCH 5/6] Icinga DB: make host problem change events update the state tables but not write state history StateChangeHandler() is the function used when the actual hard/soft state changes and thus also writes state history. This is not desired in this case, instead, a runtime update should be generated, therefore call UpdateState() instead. refs #9063 --- lib/icingadb/icingadb-objects.cpp | 9 ++++++++- lib/icingadb/icingadb.hpp | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index fbfee0b64..99e7b0434 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -119,7 +119,7 @@ void IcingaDB::ConfigStaticInitialize() }); Service::OnHostProblemChanged.connect([](const Service::Ptr& service, const CheckResult::Ptr&, const MessageOrigin::Ptr&) { - IcingaDB::StateChangeHandler(service); + IcingaDB::HostProblemChangedHandler(service); }); Notification::OnUsersRawChangedWithOldValue.connect([](const Notification::Ptr& notification, const Value& oldValues, const Value& newValues) { @@ -2696,6 +2696,13 @@ void IcingaDB::NextCheckChangedHandler(const Checkable::Ptr& checkable) } } +void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) { + for (auto& rw : ConfigType::GetObjectsByType()) { + /* Host state changes affect is_handled and severity of services. */ + rw->UpdateState(service, StateUpdate::Full); + } +} + void IcingaDB::AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry) { auto rws (ConfigType::GetObjectsByType()); diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index f0b62047f..5bbb99d04 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -152,6 +152,7 @@ private: static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime); static void NewCheckResultHandler(const Checkable::Ptr& checkable); static void NextCheckChangedHandler(const Checkable::Ptr& checkable); + static void HostProblemChangedHandler(const Service::Ptr& service); static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry); static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime); static void NotificationUsersChangedHandler(const Notification::Ptr& notification, const Array::Ptr& oldValues, const Array::Ptr& newValues); From 31da6a56e6c2f7556dcdc7a53a22a973be91a5e1 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Jan 2022 16:45:25 +0100 Subject: [PATCH 6/6] Icinga DB: remove obsolete StateChangeHandler overload This version of StateChangeHandler is no longer called anywhere as it was the wrong function for all previous callers anyways. --- lib/icingadb/icingadb-objects.cpp | 9 --------- lib/icingadb/icingadb.hpp | 1 - 2 files changed, 10 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 99e7b0434..232ac6752 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -2592,15 +2592,6 @@ IcingaDB::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, //m_Rcon->FireAndForgetQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } -void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object) -{ - auto checkable (dynamic_pointer_cast(object)); - - if (checkable) { - IcingaDB::StateChangeHandler(object, checkable->GetLastCheckResult(), checkable->GetStateType()); - } -} - void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 5bbb99d04..0ea003a34 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -136,7 +136,6 @@ private: static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj); static bool PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums); - static void StateChangeHandler(const ConfigObject::Ptr& object); static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); static void VersionChangedHandler(const ConfigObject::Ptr& object); static void DowntimeStartedHandler(const Downtime::Ptr& downtime);