tyler.durden
/
icinga2
mirror of https://github.com/Icinga/icinga2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10148 from Icinga/enhanced-sort-types-by-load-dependencies 

Sort config types by their load dependencies once
This commit is contained in:
Yonas Habteab 2024-09-26 15:27:41 +02:00 committed by GitHub
parent 0fff4153ef eb97676d69
commit 92df9ef8c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 238 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/base/initialize.hpp
View File

@ -23,6 +23,7 @@ enum class InitializePriority {
RegisterBuiltinTypes,
RegisterFunctions,
RegisterTypes,
SortTypes,
EvaluateConfigFragments,
Default,
FreezeNamespaces,

47
lib/base/type.cpp
View File

@ -1,9 +1,13 @@
/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
#include "base/type.hpp"
#include "base/atomic.hpp"
#include "base/configobject.hpp"
#include "base/debug.hpp"
#include "base/scriptglobal.hpp"
#include "base/namespace.hpp"
#include "base/objectlock.hpp"
#include <functional>
using namespace icinga;
@ -32,6 +36,43 @@ INITIALIZE_ONCE_WITH_PRIORITY([]() {
Type::Register(type);
}, InitializePriority::RegisterTypeType);
static std::vector<Type::Ptr> l_SortedByLoadDependencies;
static Atomic l_SortingByLoadDependenciesDone (false);
INITIALIZE_ONCE_WITH_PRIORITY([] {
std::unordered_set<Type*> visited;
std::function<void(Type*)> visit;
// Please note that this callback does not detect any cyclic load dependencies,
// instead, it relies on the "sort_by_load_after" unit test to fail.
visit = ([&visit, &visited](Type* type) {
if (visited.find(type) != visited.end()) {
return;
}
visited.emplace(type);
for (auto dependency : type->GetLoadDependencies()) {
visit(dependency);
}
// We have managed to reach the final/top node in this dependency graph,
// so let's place them in reverse order to their final place.
l_SortedByLoadDependencies.emplace_back(type);
});
// Sort the types by their load_after dependencies in a Depth-First search manner.
for (const Type::Ptr& type : Type::GetAllTypes()) {
// Note that only those types that are assignable to the dynamic ConfigObject type can have "load_after"
// dependencies, otherwise they are just some Icinga 2 primitive types such as Number, String, etc. and
// we need to ignore them.
if (ConfigObject::TypeInstance->IsAssignableFrom(type)) {
visit(type.get());
}
}
l_SortingByLoadDependenciesDone.store(true);
}, InitializePriority::SortTypes);
String Type::ToString() const
{
return "type '" + GetName() + "'";
@ -72,6 +113,12 @@ std::vector<Type::Ptr> Type::GetAllTypes()
return types;
}
const std::vector<Type::Ptr>& Type::GetConfigTypesSortedByLoadDependencies()
{
VERIFY(l_SortingByLoadDependenciesDone.load());
return l_SortedByLoadDependencies;
}
String Type::GetPluralName() const
{
String name = GetName();

15
lib/base/type.hpp
View File

@ -83,6 +83,21 @@ public:
static Type::Ptr GetByName(const String& name);
static std::vector<Type::Ptr> GetAllTypes();
/**
* Returns a list of config types sorted by their "load_after" dependencies.
*
* All dependencies of a given type are listed at a lower index than that of the type itself. In other words,
* if a `Service` type load depends on the `Host` and `ApiListener` types, the Host and ApiListener types are
* guaranteed to appear first on the list. Nevertheless, the order of the Host and ApiListener types themselves
* is arbitrary if the two types are not dependent.
*
* It should be noted that this method will fail fatally when used prior to the completion
* of namespace initialization.
*
* @return std::vector<Type::Ptr>
*/
static const std::vector<Ptr>& GetConfigTypesSortedByLoadDependencies();
void SetField(int id, const Value& value, bool suppress_events = false, const Value& cookie = Empty) override;
Value GetField(int id) const override;

229
lib/config/configitem.cpp
View File

@ -444,74 +444,47 @@ bool ConfigItem::CommitNewItems(const ActivationContext::Ptr& context, WorkQueue
<< "Committing " << total << " new items.";
#endif /* I2_DEBUG */
std::set<Type::Ptr> types;
std::set<Type::Ptr> completed_types;
int itemsCount {0};
for (const Type::Ptr& type : Type::GetAllTypes()) {
if (ConfigObject::TypeInstance->IsAssignableFrom(type))
types.insert(type);
}
for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
std::atomic<int> committed_items(0);
while (types.size() != completed_types.size()) {
for (const Type::Ptr& type : types) {
if (completed_types.find(type) != completed_types.end())
continue;
{
auto items (itemsByType.find(type.get()));
bool unresolved_dep = false;
/* skip this type (for now) if there are unresolved load dependencies */
for (auto pLoadDep : type->GetLoadDependencies()) {
if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
unresolved_dep = true;
break;
if (items != itemsByType.end()) {
for (const ItemPair& pair: items->second) {
newItems.emplace_back(pair.first);
}
}
if (unresolved_dep)
continue;
upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
std::atomic<int> committed_items(0);
{
auto items (itemsByType.find(type.get()));
if (items != itemsByType.end()) {
for (const ItemPair& pair: items->second) {
newItems.emplace_back(pair.first);
}
upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
if (!item->Commit(ip.second)) {
if (item->IsIgnoreOnError()) {
item->Unregister();
}
return;
if (!item->Commit(ip.second)) {
if (item->IsIgnoreOnError()) {
item->Unregister();
}
committed_items++;
});
return;
}
upq.Join();
}
committed_items++;
});
upq.Join();
}
}
itemsCount += committed_items;
completed_types.insert(type);
itemsCount += committed_items;
#ifdef I2_DEBUG
if (committed_items > 0)
Log(LogDebug, "configitem")
<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
if (committed_items > 0)
Log(LogDebug, "configitem")
<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */
if (upq.HasExceptions())
return false;
}
if (upq.HasExceptions())
return false;
}
#ifdef I2_DEBUG
@ -519,105 +492,83 @@ bool ConfigItem::CommitNewItems(const ActivationContext::Ptr& context, WorkQueue
<< "Committed " << itemsCount << " items.";
#endif /* I2_DEBUG */
completed_types.clear();
for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
std::atomic<int> notified_items(0);
while (types.size() != completed_types.size()) {
for (const Type::Ptr& type : types) {
if (completed_types.find(type) != completed_types.end())
continue;
{
auto items (itemsByType.find(type.get()));
bool unresolved_dep = false;
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
/* skip this type (for now) if there are unresolved load dependencies */
for (auto pLoadDep : type->GetLoadDependencies()) {
if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
unresolved_dep = true;
break;
}
}
if (!item->m_Object)
return;
if (unresolved_dep)
continue;
std::atomic<int> notified_items(0);
{
auto items (itemsByType.find(type.get()));
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
if (!item->m_Object)
return;
try {
item->m_Object->OnAllConfigLoaded();
notified_items++;
} catch (const std::exception& ex) {
if (!item->m_IgnoreOnError)
throw;
Log(LogNotice, "ConfigObject")
<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);
item->Unregister();
{
std::unique_lock<std::mutex> lock(item->m_Mutex);
item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
}
}
});
upq.Join();
}
}
completed_types.insert(type);
#ifdef I2_DEBUG
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */
if (upq.HasExceptions())
return false;
notified_items = 0;
for (auto loadDep : type->GetLoadDependencies()) {
auto items (itemsByType.find(loadDep));
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
if (!item->m_Object)
return;
ActivationScope ascope(item->m_ActivationContext);
item->m_Object->CreateChildObjects(type);
try {
item->m_Object->OnAllConfigLoaded();
notified_items++;
});
}
}
} catch (const std::exception& ex) {
if (!item->m_IgnoreOnError)
throw;
upq.Join();
Log(LogNotice, "ConfigObject")
<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);
item->Unregister();
{
std::unique_lock<std::mutex> lock(item->m_Mutex);
item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
}
}
});
upq.Join();
}
}
#ifdef I2_DEBUG
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */
if (upq.HasExceptions())
return false;
if (upq.HasExceptions())
return false;
// Make sure to activate any additionally generated items
if (!CommitNewItems(context, upq, newItems))
return false;
notified_items = 0;
for (auto loadDep : type->GetLoadDependencies()) {
auto items (itemsByType.find(loadDep));
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
if (!item->m_Object)
return;
ActivationScope ascope(item->m_ActivationContext);
item->m_Object->CreateChildObjects(type);
notified_items++;
});
}
}
upq.Join();
#ifdef I2_DEBUG
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */
if (upq.HasExceptions())
return false;
// Make sure to activate any additionally generated items
if (!CommitNewItems(context, upq, newItems))
return false;
}
return true;

58
test/CMakeLists.txt
View File

@ -2,6 +2,59 @@
include(BoostTestTargets)
set(types_test_SOURCES
icingaapplication-fixture.cpp
base-type.cpp
${base_OBJS}
$<TARGET_OBJECTS:config>
$<TARGET_OBJECTS:remote>
$<TARGET_OBJECTS:icinga>
$<TARGET_OBJECTS:methods>
)
if(ICINGA2_WITH_CHECKER)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:checker>)
endif()
if(ICINGA2_WITH_MYSQL)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:db_ido> $<TARGET_OBJECTS:db_ido_mysql>)
endif()
if(ICINGA2_WITH_PGSQL)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:db_ido> $<TARGET_OBJECTS:db_ido_pgsql>)
endif()
if(ICINGA2_WITH_ICINGADB)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:icingadb>)
endif()
if(ICINGA2_WITH_NOTIFICATION)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:notification>)
endif()
if(ICINGA2_WITH_PERFDATA)
list(APPEND types_test_SOURCES $<TARGET_OBJECTS:perfdata>)
endif()
if(ICINGA2_UNITY_BUILD)
mkunity_target(types test types_test_SOURCES)
endif()
# In order to test the order of all Icinga 2 config type load dependencies, we need to link against all the libraries,
# but this results in boost signals e.g. in dbevents.cpp being triggered by icinga-checkresult.cpp test cases that
# only pass partially initialised objects. Therefore, the types test cases are decoupled from base and moved to a
# separate executable to not crash the base test cases.
add_boost_test(types
SOURCES test-runner.cpp ${types_test_SOURCES}
LIBRARIES ${base_DEPS}
TESTS
types/gettype
types/assign
types/byname
types/instantiate
types/sort_by_load_after
)
set(base_test_SOURCES
icingaapplication-fixture.cpp
base-array.cpp
@ -21,7 +74,6 @@ set(base_test_SOURCES
base-string.cpp
base-timer.cpp
base-tlsutility.cpp
base-type.cpp
base-utility.cpp
base-value.cpp
config-apply.cpp
@ -117,10 +169,6 @@ add_boost_test(base
base_tlsutility/iscertuptodate_ok
base_tlsutility/iscertuptodate_expiring
base_tlsutility/iscertuptodate_old
base_type/gettype
base_type/assign
base_type/byname
base_type/instantiate
base_utility/parse_version
base_utility/compare_version
base_utility/comparepasswords_works

33
test/base-type.cpp
View File

@ -5,11 +5,13 @@
#include "base/objectlock.hpp"
#include "base/application.hpp"
#include "base/type.hpp"
#include "icinga/host.hpp"
#include "icinga/service.hpp"
#include <BoostTestTargetConfig.h>
using namespace icinga;
BOOST_AUTO_TEST_SUITE(base_type)
BOOST_AUTO_TEST_SUITE(types)
BOOST_AUTO_TEST_CASE(gettype)
{
@ -44,4 +46,33 @@ BOOST_AUTO_TEST_CASE(instantiate)
BOOST_CHECK(p);
}
BOOST_AUTO_TEST_CASE(sort_by_load_after)
{
int totalDependencies{0};
std::unordered_set<Type*> previousTypes;
for (auto type : Type::GetConfigTypesSortedByLoadDependencies()) {
BOOST_CHECK_EQUAL(true, ConfigObject::TypeInstance->IsAssignableFrom(type));
totalDependencies += type->GetLoadDependencies().size();
for (Type* dependency : type->GetLoadDependencies()) {
// Note, Type::GetConfigTypesSortedByLoadDependencies() does not detect any cyclic load dependencies,
// instead, it relies on this test case to fail.
BOOST_CHECK_MESSAGE(previousTypes.find(dependency) != previousTypes.end(), "type '" << type->GetName()
<< "' depends on '"<< dependency->GetName() << "' type, but it's not loaded before");
}
previousTypes.emplace(type.get());
}
// The magic number 12 is the sum of host,service,comment,downtime and endpoint load_after dependencies.
BOOST_CHECK_MESSAGE(totalDependencies >= 12, "total size of load dependency must be at least 12");
BOOST_CHECK_MESSAGE(previousTypes.find(Host::TypeInstance.get()) != previousTypes.end(), "Host type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Service::TypeInstance.get()) != previousTypes.end(), "Service type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Downtime::TypeInstance.get()) != previousTypes.end(), "Downtime type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Comment::TypeInstance.get()) != previousTypes.end(), "Comment type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Notification::TypeInstance.get()) != previousTypes.end(), "Notification type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Zone::TypeInstance.get()) != previousTypes.end(), "Zone type should be in the list");
BOOST_CHECK_MESSAGE(previousTypes.find(Endpoint::TypeInstance.get()) != previousTypes.end(), "Endpoint type should be in the list");
}
BOOST_AUTO_TEST_SUITE_END()