From 2c39d69428f9c7b5a7ef8ea05a848984b65938e5 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 17:12:02 +0200 Subject: [PATCH 01/37] Implement first draft for cluster config staged sync --- lib/remote/CMakeLists.txt | 1 + lib/remote/apilistener-filesync.cpp | 88 +++++++++++++++++++++++++---- lib/remote/apilistener.cpp | 10 ++++ lib/remote/apilistener.hpp | 12 +++- 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/lib/remote/CMakeLists.txt b/lib/remote/CMakeLists.txt index da0006aa1..2c5a0326a 100644 --- a/lib/remote/CMakeLists.txt +++ b/lib/remote/CMakeLists.txt @@ -59,6 +59,7 @@ set_target_properties ( #install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api\")") install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/log\")") install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/zones\")") +install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/zones-stage\")") install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/certs\")") install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/certificate-requests\")") diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 411028630..03fc68475 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -62,7 +62,8 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) return config; } -bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, const String& configDir, bool authoritative) +bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, + const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative) { bool configChange = false; @@ -105,6 +106,9 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con if (!Utility::Match("*/.timestamp", kv.first)) configChange = true; + /* Store the relative config file path for later. */ + relativePaths.push_back(zoneName + "/" + kv.first); + String path = configDir + "/" + kv.first; Log(LogInformation, "ApiListener") << "Updating configuration file: " << path; @@ -123,6 +127,7 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con } } + /* Update with staging information TODO - use `authoritative` as flag. */ Log(LogInformation, "ApiListener") << "Applying configuration file update for path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" @@ -197,7 +202,8 @@ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir); - UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, true); + std::vector relativePaths; + UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, zone->GetName(), relativePaths, true); } void ApiListener::SyncZoneDirs() const @@ -289,26 +295,34 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Dictionary::Ptr updateV2 = params->Get("update_v2"); bool configChange = false; + std::vector relativePaths; ObjectLock olock(updateV1); for (const Dictionary::Pair& kv : updateV1) { + + /* Check for the configured zones. */ Zone::Ptr zone = Zone::GetByName(kv.first); + String zoneName = zone->GetName(); if (!zone) { Log(LogWarning, "ApiListener") - << "Ignoring config update for unknown zone '" << kv.first << "'."; + << "Ignoring config update for unknown zone '" << zoneName << "'."; continue; } + /* Whether we already have configuration in zones.d. */ if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) { Log(LogWarning, "ApiListener") - << "Ignoring config update for zone '" << kv.first << "' because we have an authoritative version of the zone's config."; + << "Ignoring config update for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; continue; } - String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName(); + /* Put the received configuration into our stage directory. */ + String currentConfigDir = GetApiZonesDir() + zoneName; + String stageConfigDir = GetApiZonesStageDir() + zoneName; - Utility::MkDirP(oldDir, 0700); + Utility::MkDirP(currentConfigDir, 0700); + Utility::MkDirP(stageConfigDir, 0700); ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = kv.second; @@ -317,16 +331,70 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D newConfigInfo.UpdateV2 = updateV2->Get(kv.first); Dictionary::Ptr newConfig = kv.second; - ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir); + ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); - if (UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, false)) + /* Move the received configuration into our stage directory first. */ + if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) configChange = true; } if (configChange) { - Log(LogInformation, "ApiListener", "Restarting after configuration change."); - Application::RequestRestart(); + /* Spawn a validation process. On success, move the staged configuration + * into production and restart. + */ + AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths, true); } return Empty; } + +void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, + const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths, bool reload) +{ + String logFile = GetApiZonesStageDir() + "/startup.log"; + std::ofstream fpLog(logFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fpLog << pr.Output; + fpLog.close(); + + String statusFile = GetApiZonesStageDir() + "/status"; + std::ofstream fpStatus(statusFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fpStatus << pr.ExitStatus; + fpStatus.close(); + + /* validation went fine, copy stage and reload */ + if (pr.ExitStatus == 0) { + for (const String& path : relativePaths) { + /* TODO: Better error handling with existing files. */ + Log(LogCritical, "ApiListener") + << "Copying file '" << path << "' from config sync staging to production directory."; + + Utility::CopyFile(GetApiZonesStageDir() + path, GetApiZonesDir() + path); + } + + if (reload) + Application::RequestRestart(); + } else { + Log(LogCritical, "ApiListener") + << "Config validation failed for staged cluster config sync. Stage not put in production, aborting."; + } +} + +void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths, bool reload) +{ + VERIFY(Application::GetArgC() >= 1); + + // prepare arguments + Array::Ptr args = new Array({ + Application::GetExePath(Application::GetArgV()[0]), + "daemon", + "--validate", + "--define", + "ZonesDir=" + GetApiZonesStageDir() + }); + + Process::Ptr process = new Process(Process::PrepareCommand(args)); + process->SetTimeout(300); + process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths, reload)); +} diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index a7f0f66a7..b9f8b9780 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -59,6 +59,16 @@ String ApiListener::GetApiDir() return Configuration::DataDir + "/api/"; } +String ApiListener::GetApiZonesDir() +{ + return GetApiDir() + "zones/"; +} + +String ApiListener::GetApiZonesStageDir() +{ + return GetApiDir() + "zones-stage/"; +} + String ApiListener::GetCertsDir() { return Configuration::DataDir + "/certs/"; diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 022ad8281..e8512cd6b 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -9,6 +9,7 @@ #include "remote/endpoint.hpp" #include "remote/messageorigin.hpp" #include "base/configobject.hpp" +#include "base/process.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" #include "base/tcpsocket.hpp" @@ -47,6 +48,8 @@ public: ApiListener(); static String GetApiDir(); + static String GetApiZonesDir(); + static String GetApiZonesStageDir(); static String GetCertsDir(); static String GetCaDir(); static String GetCertificateRequestsDir(); @@ -169,7 +172,8 @@ private: static ConfigDirInformation LoadConfigDir(const String& dir); static Dictionary::Ptr MergeConfigUpdate(const ConfigDirInformation& config); - static bool UpdateConfigDir(const ConfigDirInformation& oldConfig, const ConfigDirInformation& newConfig, const String& configDir, bool authoritative); + static bool UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, + const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative); void SyncZoneDirs() const; void SyncZoneDir(const Zone::Ptr& zone) const; @@ -177,6 +181,12 @@ private: static void ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file); void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient); + static void TryActivateZonesStageCallback(const ProcessResult& pr, + const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths, bool reload); + static void AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths, bool reload); + /* configsync */ void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin, const JsonRpcConnection::Ptr& client = nullptr); From 506eee2f7dc3f79f791188e3e0ede8b875e34889 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 17:55:54 +0200 Subject: [PATCH 02/37] Fix crash --- lib/remote/apilistener-filesync.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 03fc68475..68b96b9df 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -301,8 +301,8 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D for (const Dictionary::Pair& kv : updateV1) { /* Check for the configured zones. */ - Zone::Ptr zone = Zone::GetByName(kv.first); - String zoneName = zone->GetName(); + String zoneName = kv.first; + Zone::Ptr zone = Zone::GetByName(zoneName); if (!zone) { Log(LogWarning, "ApiListener") @@ -311,7 +311,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D } /* Whether we already have configuration in zones.d. */ - if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) { + if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { Log(LogWarning, "ApiListener") << "Ignoring config update for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; continue; From c2d7063ae726938c5a1037a77c7fe734a0a71be9 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 18:19:48 +0200 Subject: [PATCH 03/37] Better signal for checking the cluster config sync stage (ignore production) --- lib/cli/daemonutility.cpp | 17 ++++++++++++----- lib/remote/apilistener-filesync.cpp | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/cli/daemonutility.cpp b/lib/cli/daemonutility.cpp index b586544ee..802f1e758 100644 --- a/lib/cli/daemonutility.cpp +++ b/lib/cli/daemonutility.cpp @@ -100,6 +100,10 @@ static void IncludePackage(const String& packagePath, bool& success) bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, const String& objectsFile) { bool success; + + Namespace::Ptr systemNS = ScriptGlobal::Get("System"); + VERIFY(systemNS); + if (!objectsFile.IsEmpty()) ConfigCompilerContext::GetInstance()->OpenObjectsFile(objectsFile); @@ -138,17 +142,20 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, if (!success) return false; - /* Load cluster synchronized configuration files */ + /* Load cluster synchronized configuration files. This can be disabled for staged sync validations. */ + bool ignoreZonesVarDir = false; + if (systemNS->Contains("IgnoreZonesVarDir")) { + ignoreZonesVarDir = Convert::ToBool(systemNS->Get("IgnoreZonesVarDir")); + } + String zonesVarDir = Configuration::DataDir + "/api/zones"; - if (Utility::PathExists(zonesVarDir)) + + if (!ignoreZonesVarDir && Utility::PathExists(zonesVarDir)) Utility::Glob(zonesVarDir + "/*", std::bind(&IncludeNonLocalZone, _1, "_cluster", std::ref(success)), GlobDirectory); if (!success) return false; - Namespace::Ptr systemNS = ScriptGlobal::Get("System"); - VERIFY(systemNS); - /* This is initialized inside the IcingaApplication class. */ Value vAppType; VERIFY(systemNS->Get("ApplicationType", &vAppType)); diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 68b96b9df..f1d590293 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -391,7 +391,7 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const "daemon", "--validate", "--define", - "ZonesDir=" + GetApiZonesStageDir() + "System.IgnoreZonesVarDir=true" }); Process::Ptr process = new Process(Process::PrepareCommand(args)); From 2ed56b50a4164aa6f4c039187991732d41e4b52d Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 18:31:08 +0200 Subject: [PATCH 04/37] Ensure directory paths are created from stage -> prod --- lib/remote/apilistener-filesync.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index f1d590293..1ae461d74 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -369,7 +369,12 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Log(LogCritical, "ApiListener") << "Copying file '" << path << "' from config sync staging to production directory."; - Utility::CopyFile(GetApiZonesStageDir() + path, GetApiZonesDir() + path); + String stagePath = GetApiZonesStageDir() + path; + String currentPath = GetApiZonesDir() + path; + + Utility::MkDirP(Utility::DirName(currentPath), 0755); + + Utility::CopyFile(stagePath, currentPath); } if (reload) From 18532542014bdb4518433ee2d607d38922de4751 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 18:49:49 +0200 Subject: [PATCH 05/37] Pass the zonesVar override around --- lib/cli/daemonutility.cpp | 12 ++++++------ lib/remote/apilistener-filesync.cpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cli/daemonutility.cpp b/lib/cli/daemonutility.cpp index 802f1e758..fcf0cc50d 100644 --- a/lib/cli/daemonutility.cpp +++ b/lib/cli/daemonutility.cpp @@ -143,14 +143,14 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, return false; /* Load cluster synchronized configuration files. This can be disabled for staged sync validations. */ - bool ignoreZonesVarDir = false; - if (systemNS->Contains("IgnoreZonesVarDir")) { - ignoreZonesVarDir = Convert::ToBool(systemNS->Get("IgnoreZonesVarDir")); - } - String zonesVarDir = Configuration::DataDir + "/api/zones"; - if (!ignoreZonesVarDir && Utility::PathExists(zonesVarDir)) + /* Cluster config sync stage validation needs this. */ + if (systemNS->Contains("ZonesStageVarDir")) { + zonesVarDir = systemNS->Get("ZonesStageVarDir"); + } + + if (Utility::PathExists(zonesVarDir)) Utility::Glob(zonesVarDir + "/*", std::bind(&IncludeNonLocalZone, _1, "_cluster", std::ref(success)), GlobDirectory); if (!success) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 1ae461d74..52203f9d1 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -396,7 +396,7 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const "daemon", "--validate", "--define", - "System.IgnoreZonesVarDir=true" + "System.ZonesStageVarDir='" + GetApiZonesStageDir() + "'" //Path is written onto the shell }); Process::Ptr process = new Process(Process::PrepareCommand(args)); From e545884952de42badf224e8762a2a3e0650f855a Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 19:49:16 +0200 Subject: [PATCH 06/37] Improve logging for staged config sync --- lib/cli/daemonutility.cpp | 16 +++++++++++----- lib/remote/apilistener-filesync.cpp | 6 +++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/cli/daemonutility.cpp b/lib/cli/daemonutility.cpp index fcf0cc50d..f5b436bda 100644 --- a/lib/cli/daemonutility.cpp +++ b/lib/cli/daemonutility.cpp @@ -126,12 +126,15 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, * unfortunately moving it there is somewhat non-trivial. */ success = true; - String zonesEtcDir = Configuration::ZonesDir; - if (!zonesEtcDir.IsEmpty() && Utility::PathExists(zonesEtcDir)) - Utility::Glob(zonesEtcDir + "/*", std::bind(&IncludeZoneDirRecursive, _1, "_etc", std::ref(success)), GlobDirectory); + /* Only load zone directory if we're not in staging validation. */ + if (!systemNS->Contains("ZonesStageVarDir")) { + String zonesEtcDir = Configuration::ZonesDir; + if (!zonesEtcDir.IsEmpty() && Utility::PathExists(zonesEtcDir)) + Utility::Glob(zonesEtcDir + "/*", std::bind(&IncludeZoneDirRecursive, _1, "_etc", std::ref(success)), GlobDirectory); - if (!success) - return false; + if (!success) + return false; + } /* Load package config files - they may contain additional zones which * are authoritative on this node and are checked in HasZoneConfigAuthority(). */ @@ -148,6 +151,9 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, /* Cluster config sync stage validation needs this. */ if (systemNS->Contains("ZonesStageVarDir")) { zonesVarDir = systemNS->Get("ZonesStageVarDir"); + + Log(LogDebug, "DaemonUtility") + << "Overriding zones var directory with '" << zonesVarDir << "' for cluster config sync staging."; } if (Utility::PathExists(zonesVarDir)) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 52203f9d1..8fe90ee43 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -364,6 +364,9 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* validation went fine, copy stage and reload */ if (pr.ExitStatus == 0) { + Log(LogInformation, "ApiListener") + << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, triggering reload."; + for (const String& path : relativePaths) { /* TODO: Better error handling with existing files. */ Log(LogCritical, "ApiListener") @@ -381,7 +384,8 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Application::RequestRestart(); } else { Log(LogCritical, "ApiListener") - << "Config validation failed for staged cluster config sync. Stage not put in production, aborting."; + << "Config validation failed for staged cluster config sync. Aborting. Logs: '" + << GetApiZonesStageDir() + "/startup.log'"; } } From e3e68caaa36c9147ac4c4d1cc284cd5213efbd02 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 20:05:25 +0200 Subject: [PATCH 07/37] Inherit parent process arguments for defined path constants --- lib/remote/apilistener-filesync.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 8fe90ee43..87f1be475 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -394,15 +394,24 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const { VERIFY(Application::GetArgC() >= 1); - // prepare arguments + /* Inherit parent process args. */ Array::Ptr args = new Array({ Application::GetExePath(Application::GetArgV()[0]), - "daemon", - "--validate", - "--define", - "System.ZonesStageVarDir='" + GetApiZonesStageDir() + "'" //Path is written onto the shell }); + for (int i = 1; i < Application::GetArgC(); i++) { + String argV = Application::GetArgV()[i]; + + if (argV == "-d" || argV == "--daemonize") + continue; + + args->Add(argV); + } + + args->Add("--validate"); + args->Add("--define"); + args->Add("System.ZonesStageVarDir='" + GetApiZonesStageDir() + "'"); + Process::Ptr process = new Process(Process::PrepareCommand(args)); process->SetTimeout(300); process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths, reload)); From a91bbe8acdb015b2c3942704e1152c153d467665 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 27 Sep 2018 20:13:54 +0200 Subject: [PATCH 08/37] Fix constant value for zone var override --- lib/cli/daemonutility.cpp | 3 ++- lib/remote/apilistener-filesync.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cli/daemonutility.cpp b/lib/cli/daemonutility.cpp index f5b436bda..5508b1488 100644 --- a/lib/cli/daemonutility.cpp +++ b/lib/cli/daemonutility.cpp @@ -152,10 +152,11 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, if (systemNS->Contains("ZonesStageVarDir")) { zonesVarDir = systemNS->Get("ZonesStageVarDir"); - Log(LogDebug, "DaemonUtility") + Log(LogInformation, "DaemonUtility") << "Overriding zones var directory with '" << zonesVarDir << "' for cluster config sync staging."; } + if (Utility::PathExists(zonesVarDir)) Utility::Glob(zonesVarDir + "/*", std::bind(&IncludeNonLocalZone, _1, "_cluster", std::ref(success)), GlobDirectory); diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 87f1be475..5de7df299 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -410,7 +410,7 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const args->Add("--validate"); args->Add("--define"); - args->Add("System.ZonesStageVarDir='" + GetApiZonesStageDir() + "'"); + args->Add("System.ZonesStageVarDir=" + GetApiZonesStageDir()); Process::Ptr process = new Process(Process::PrepareCommand(args)); process->SetTimeout(300); From fb367e12cc1c226af2f515d9072d33ada21daf1e Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 28 Sep 2018 11:08:58 +0200 Subject: [PATCH 09/37] Store the last failed zone stage sync validation as runtime ApiListener attribute --- lib/remote/apilistener-filesync.cpp | 17 ++++++++++++++++- lib/remote/apilistener.hpp | 2 ++ lib/remote/apilistener.ti | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 5de7df299..f851153fe 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -385,7 +385,12 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, } else { Log(LogCritical, "ApiListener") << "Config validation failed for staged cluster config sync. Aborting. Logs: '" - << GetApiZonesStageDir() + "/startup.log'"; + << logFile << "'"; + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) + listener->UpdateLastFailedZonesStageValidation(pr.Output); } } @@ -416,3 +421,13 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const process->SetTimeout(300); process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths, reload)); } + +void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) +{ + Dictionary::Ptr lastFailedZonesStageValidation = new Dictionary({ + { "log", log }, + { "ts", Utility::GetTime() } + }); + + SetLastFailedZonesStageValidation(lastFailedZonesStageValidation); +} diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index e8512cd6b..757de7e87 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -187,6 +187,8 @@ private: static void AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, const std::vector& relativePaths, bool reload); + void UpdateLastFailedZonesStageValidation(const String& log); + /* configsync */ void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin, const JsonRpcConnection::Ptr& client = nullptr); diff --git a/lib/remote/apilistener.ti b/lib/remote/apilistener.ti index 4217ce0ab..ac17ccfcf 100644 --- a/lib/remote/apilistener.ti +++ b/lib/remote/apilistener.ti @@ -54,6 +54,8 @@ class ApiListener : ConfigObject [state, no_user_modify] Timestamp log_message_timestamp; [no_user_modify] String identity; + + [state, no_user_modify] Dictionary::Ptr last_failed_zones_stage_validation; }; } From 86108e6a1e74c0caca43587a4df818db23839912 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 28 Sep 2018 11:26:47 +0200 Subject: [PATCH 10/37] Improve logging and code quality --- lib/cli/daemonutility.cpp | 4 ++-- lib/remote/apilistener-filesync.cpp | 22 ++++++++++------------ lib/remote/apilistener.hpp | 4 ++-- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/cli/daemonutility.cpp b/lib/cli/daemonutility.cpp index 5508b1488..d5bb2cf1e 100644 --- a/lib/cli/daemonutility.cpp +++ b/lib/cli/daemonutility.cpp @@ -145,14 +145,14 @@ bool DaemonUtility::ValidateConfigFiles(const std::vector& configs, if (!success) return false; - /* Load cluster synchronized configuration files. This can be disabled for staged sync validations. */ + /* Load cluster synchronized configuration files. This can be overridden for staged sync validations. */ String zonesVarDir = Configuration::DataDir + "/api/zones"; /* Cluster config sync stage validation needs this. */ if (systemNS->Contains("ZonesStageVarDir")) { zonesVarDir = systemNS->Get("ZonesStageVarDir"); - Log(LogInformation, "DaemonUtility") + Log(LogNotice, "DaemonUtility") << "Overriding zones var directory with '" << zonesVarDir << "' for cluster config sync staging."; } diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index f851153fe..4d8351572 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -342,7 +342,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D /* Spawn a validation process. On success, move the staged configuration * into production and restart. */ - AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths, true); + AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths); } return Empty; @@ -350,7 +350,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths, bool reload) + const std::vector& relativePaths) { String logFile = GetApiZonesStageDir() + "/startup.log"; std::ofstream fpLog(logFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); @@ -365,12 +365,11 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* validation went fine, copy stage and reload */ if (pr.ExitStatus == 0) { Log(LogInformation, "ApiListener") - << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, triggering reload."; + << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, copying into '" << GetApiZonesDir() << "' and triggering reload."; for (const String& path : relativePaths) { - /* TODO: Better error handling with existing files. */ - Log(LogCritical, "ApiListener") - << "Copying file '" << path << "' from config sync staging to production directory."; + Log(LogNotice, "ApiListener") + << "Copying file '" << path << "' from config sync staging to production zones directory."; String stagePath = GetApiZonesStageDir() + path; String currentPath = GetApiZonesDir() + path; @@ -380,12 +379,11 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Utility::CopyFile(stagePath, currentPath); } - if (reload) - Application::RequestRestart(); + Application::RequestRestart(); } else { Log(LogCritical, "ApiListener") - << "Config validation failed for staged cluster config sync. Aborting. Logs: '" - << logFile << "'"; + << "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir() + << "'. Aborting. Logs: '" << logFile << "'"; ApiListener::Ptr listener = ApiListener::GetInstance(); @@ -395,7 +393,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, } void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths, bool reload) + const std::vector& relativePaths) { VERIFY(Application::GetArgC() >= 1); @@ -419,7 +417,7 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const Process::Ptr process = new Process(Process::PrepareCommand(args)); process->SetTimeout(300); - process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths, reload)); + process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths)); } void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 757de7e87..3be03f96d 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -183,9 +183,9 @@ private: static void TryActivateZonesStageCallback(const ProcessResult& pr, const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths, bool reload); + const std::vector& relativePaths); static void AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths, bool reload); + const std::vector& relativePaths); void UpdateLastFailedZonesStageValidation(const String& log); From 9d53db1401af79eca7e22b94491fdb904750d139 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 28 Sep 2018 13:54:54 +0200 Subject: [PATCH 11/37] Purge stage and production directories before copying files The cluster-message -> production diff is still intact, we're just taking care of unwanted/deleted files here. --- lib/remote/apilistener-filesync.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 4d8351572..004a29bad 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -136,6 +136,7 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", oldTimestamp) << "' (" << oldTimestamp << ")."; + /* TODO: Deal with recursive directories. */ ObjectLock xlock(oldConfig); for (const Dictionary::Pair& kv : oldConfig) { if (!newConfig->Contains(kv.first)) { @@ -297,6 +298,14 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D bool configChange = false; std::vector relativePaths; + /* + * We can and must safely purge the staging directory, as the difference is taken between + * runtime production config and newly received configuration. + */ + String apiZonesStageDir = GetApiZonesStageDir(); + Utility::RemoveDirRecursive(apiZonesStageDir); + Utility::MkDirP(apiZonesStageDir, 0700); + ObjectLock olock(updateV1); for (const Dictionary::Pair& kv : updateV1) { @@ -333,7 +342,9 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Dictionary::Ptr newConfig = kv.second; ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); - /* Move the received configuration into our stage directory first. */ + /* Diff the current production configuration with the received configuration. + * If there was a change, collect a signal for later stage validation. + */ if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) configChange = true; } @@ -365,8 +376,15 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* validation went fine, copy stage and reload */ if (pr.ExitStatus == 0) { Log(LogInformation, "ApiListener") - << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, copying into '" << GetApiZonesDir() << "' and triggering reload."; + << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, replacing into '" << GetApiZonesDir() << "' and triggering reload."; + String apiZonesDir = GetApiZonesDir(); + + /* Purge production before copying stage. */ + Utility::RemoveDirRecursive(apiZonesDir); + Utility::MkDirP(apiZonesDir, 0700); + + /* Copy all synced configuration files from stage to production. */ for (const String& path : relativePaths) { Log(LogNotice, "ApiListener") << "Copying file '" << path << "' from config sync staging to production zones directory."; @@ -374,7 +392,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, String stagePath = GetApiZonesStageDir() + path; String currentPath = GetApiZonesDir() + path; - Utility::MkDirP(Utility::DirName(currentPath), 0755); + Utility::MkDirP(Utility::DirName(currentPath), 0700); Utility::CopyFile(stagePath, currentPath); } From 4e9439f2d818508f7d04b55d73a771f7341a08d0 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 28 Sep 2018 16:51:20 +0200 Subject: [PATCH 12/37] Ensure that config master zones.d -> var-api-zones sync removes deleted files --- lib/remote/apilistener-filesync.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 004a29bad..ffddb51f5 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -167,11 +167,16 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const { + if (!zone) + return; + ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = new Dictionary(); newConfigInfo.UpdateV2 = new Dictionary(); - for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zone->GetName())) { + String zoneName = zone->GetName(); + + for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zoneName)) { ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path); { @@ -194,17 +199,19 @@ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const if (sumUpdates == 0) return; - String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName(); + String currentDir = Configuration::DataDir + "/api/zones/" + zoneName; Log(LogInformation, "ApiListener") - << "Copying " << sumUpdates << " zone configuration files for zone '" << zone->GetName() << "' to '" << oldDir << "'."; + << "Copying " << sumUpdates << " zone configuration files for zone '" << zoneName << "' to '" << currentDir << "'."; - Utility::MkDirP(oldDir, 0700); + ConfigDirInformation oldConfigInfo = LoadConfigDir(currentDir); - ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir); + /* Purge files to allow deletion via zones.d. */ + Utility::RemoveDirRecursive(currentDir); + Utility::MkDirP(currentDir, 0700); std::vector relativePaths; - UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, zone->GetName(), relativePaths, true); + UpdateConfigDir(oldConfigInfo, newConfigInfo, currentDir, zoneName, relativePaths, true); } void ApiListener::SyncZoneDirs() const From 83c11962b206d2bbf939b8ca5ca58ac52267d610 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 28 Sep 2018 16:58:59 +0200 Subject: [PATCH 13/37] Only remove directories if they exist during sync --- lib/remote/apilistener-filesync.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index ffddb51f5..c8afd43af 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -310,7 +310,10 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D * runtime production config and newly received configuration. */ String apiZonesStageDir = GetApiZonesStageDir(); - Utility::RemoveDirRecursive(apiZonesStageDir); + + if (Utility::PathExists(apiZonesStageDir)) + Utility::RemoveDirRecursive(apiZonesStageDir); + Utility::MkDirP(apiZonesStageDir, 0700); ObjectLock olock(updateV1); @@ -388,7 +391,9 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, String apiZonesDir = GetApiZonesDir(); /* Purge production before copying stage. */ - Utility::RemoveDirRecursive(apiZonesDir); + if (Utility::PathExists(apiZonesDir)) + Utility::RemoveDirRecursive(apiZonesDir); + Utility::MkDirP(apiZonesDir, 0700); /* Copy all synced configuration files from stage to production. */ From 46cb806b3fcf4fafc3d927781fb29aba5f3cbaef Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 23 Oct 2018 16:07:08 +0200 Subject: [PATCH 14/37] Add a note for config updates V1 and V2 Old clients sync !.conf via update_v2 message, we cannot remove this handling for the time being. --- lib/remote/apilistener-filesync.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index c8afd43af..68625a4b7 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -32,6 +32,11 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& Dictionary::Ptr update; + /* + * 'update' messages contain conf files. 'update_v2' syncs everything else (.timestamp). + * + * **Keep this intact to stay compatible with older clients.** + */ if (Utility::Match("*.conf", file)) update = config.UpdateV1; else From 2acf3a6941cf046cea15601a1a97bb11f6902d4a Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 23 Oct 2018 17:23:34 +0200 Subject: [PATCH 15/37] Indicate a warning in the 'icinga' check when cluster stage validation failed - success: clear the last failed attribute - failed: populate it with the output and current timestamp This can be used to highlight this in the 'icinga' check task. Since 2.9 we don't have problems with circular library dependencies with just one linked binary, therefore it is safe to include libremote in libmethods here. --- lib/methods/icingachecktask.cpp | 15 +++++++++++++++ lib/remote/apilistener-filesync.cpp | 27 ++++++++++++++++++++------- lib/remote/apilistener.hpp | 1 + 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/methods/icingachecktask.cpp b/lib/methods/icingachecktask.cpp index 52213889e..fa598d487 100644 --- a/lib/methods/icingachecktask.cpp +++ b/lib/methods/icingachecktask.cpp @@ -8,6 +8,7 @@ #include "icinga/icingaapplication.hpp" #include "icinga/clusterevents.hpp" #include "icinga/checkable.hpp" +#include "remote/apilistener.hpp" #include "base/application.hpp" #include "base/objectlock.hpp" #include "base/utility.hpp" @@ -157,6 +158,20 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes cr->SetState(ServiceWarning); } + /* Indicate a warning when the last synced config caused a stage validation error. */ + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) { + Dictionary::Ptr validationResult = listener->GetLastFailedZonesStageValidation(); + + if (validationResult) { + output += "; Last zone sync stage validation failed at " + + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", validationResult->Get("ts")); + + cr->SetState(ServiceWarning); + } + } + /* Extract the version number of the running Icinga2 instance. * We assume that appVersion will allways be something like 'v2.10.1-8-gaebe6da' and we want to extract '2.10.1'. */ diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 68625a4b7..e4ed1f310 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -414,17 +414,25 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Utility::CopyFile(stagePath, currentPath); } - Application::RequestRestart(); - } else { - Log(LogCritical, "ApiListener") - << "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir() - << "'. Aborting. Logs: '" << logFile << "'"; - ApiListener::Ptr listener = ApiListener::GetInstance(); if (listener) - listener->UpdateLastFailedZonesStageValidation(pr.Output); + listener->ClearLastFailedZonesStageValidation(); + + Application::RequestRestart(); + + return; } + + /* Error case. */ + Log(LogCritical, "ApiListener") + << "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir() + << "'. Aborting. Logs: '" << logFile << "'"; + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) + listener->UpdateLastFailedZonesStageValidation(pr.Output); } void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, @@ -464,3 +472,8 @@ void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) SetLastFailedZonesStageValidation(lastFailedZonesStageValidation); } + +void ApiListener::ClearLastFailedZonesStageValidation() +{ + SetLastFailedZonesStageValidation(Dictionary::Ptr()); +} diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 3be03f96d..59e73ec96 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -188,6 +188,7 @@ private: const std::vector& relativePaths); void UpdateLastFailedZonesStageValidation(const String& log); + void ClearLastFailedZonesStageValidation(); /* configsync */ void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin, From 604a8a041dfb06135bda7f9c324613345403169e Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 23 Oct 2018 18:29:53 +0200 Subject: [PATCH 16/37] Update log message and implement recursive diff delete --- lib/remote/apilistener-filesync.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index e4ed1f310..b382ffed3 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -132,23 +132,24 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con } } - /* Update with staging information TODO - use `authoritative` as flag. */ + /* Log something whether we're authoritative or receing a staged config. */ Log(LogInformation, "ApiListener") - << "Applying configuration file update for path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" + << "Applying configuration file update for " << (authoritative ? "" : "stage ") + << "path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" << std::fixed << std::setprecision(6) << newTimestamp << "), Current timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", oldTimestamp) << "' (" << oldTimestamp << ")."; - /* TODO: Deal with recursive directories. */ + /* If the update removes a path, delete it on disk. */ ObjectLock xlock(oldConfig); for (const Dictionary::Pair& kv : oldConfig) { if (!newConfig->Contains(kv.first)) { configChange = true; String path = configDir + "/" + kv.first; - (void) unlink(path.CStr()); + Utility::RemoveDirRecursive(path); } } From b3fa51a5dcb232d7e490416ab43962c9b0a318f8 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 23 Oct 2018 18:30:19 +0200 Subject: [PATCH 17/37] Code Documentation: Config file sync Adds headers to all functions including parameters. This unveils certain unused ones too. --- lib/remote/apilistener-filesync.cpp | 108 +++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index b382ffed3..79efefe37 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -17,6 +17,14 @@ REGISTER_APIFUNCTION(Update, config, &ApiListener::ConfigUpdateHandler); boost::mutex ApiListener::m_ConfigSyncStageLock; +/** + * Read the given file and store it in the config information structure. + * Callback function for Glob(). + * + * @param config Reference to the config information object. + * @param path File path. + * @param file Full file name. + */ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) { CONTEXT("Creating config update for file '" + file + "'"); @@ -45,6 +53,12 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& update->Set(file.SubStr(path.GetLength()), content); } +/** + * Compatibility helper for merging config update v1 and v2 into a global result. + * + * @param config Config information structure. + * @returns Dictionary which holds the merged information. + */ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& config) { Dictionary::Ptr result = new Dictionary(); @@ -58,6 +72,12 @@ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& confi return result; } +/** + * Load the given config dir and read their file content into the config structure. + * + * @param dir Path to the config directory. + * @returns ConfigInformation structure. + */ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) { ConfigDirInformation config; @@ -67,6 +87,23 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) return config; } +/** + * Diffs the old current configuration with the new configuration + * and copies the collected content. Detects whether a change + * happened, this is used for later restarts. + * + * This generic function is called in two situations: + * - Local zones.d to var/lib/api/zones copy on the master (authoritative: true) + * - Received config update on a cluster node (authoritative: false) + * + * @param oldConfigInfo Config information struct for the current old deployed config. + * @param newConfigInfo Config information struct for the received synced config. + * @param configDir Destination for copying new files (production, or stage dir). + * @param zoneName Currently processed zone, for storing the relative paths for later. + * @param relativePaths Reference which stores all updated config path destinations. + * @param Whether we're authoritative for this config. + * @returns Whether a config change happened. + */ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative) { @@ -171,6 +208,17 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con return configChange; } +/** + * Sync a zone directory where we have an authoritative copy (zones.d, etc.) + * + * This function collects the registered zone config dirs from + * the config compiler and reads the file content into the config + * information structure. + * + * Returns early when there are no updates. + * + * @param zone Pointer to the zone object being synced. + */ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const { if (!zone) @@ -220,6 +268,10 @@ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const UpdateConfigDir(oldConfigInfo, newConfigInfo, currentDir, zoneName, relativePaths, true); } +/** + * Entrypoint for updating all authoritative configs into var/lib/icinga2/api/zones + * + */ void ApiListener::SyncZoneDirs() const { for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { @@ -231,6 +283,14 @@ void ApiListener::SyncZoneDirs() const } } +/** + * Entrypoint for sending a file based config update to a cluster client. + * This includes security checks for zone relations. + * Loads the zone config files where this client belongs to + * and sends the 'config::Update' JSON-RPC message. + * + * @param aclient Connected JSON-RPC client. + */ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) { Endpoint::Ptr endpoint = aclient->GetEndpoint(); @@ -278,8 +338,21 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) aclient->SendMessage(message); } +/** + * Registered handler when a new config::Update message is received. + * + * Checks destination and permissions first, then analyses the update. + * The newly received configuration is not copied to production immediately, + * but into the staging directory first. + * Last, the async validation and restart is triggered. + * + * @param origin Where this message came from. + * @param params Message parameters including the config updates. + * @returns Empty, required by the interface. + */ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { + /* Verify permissions and trust relationship. */ if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) return Empty; @@ -322,6 +395,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Utility::MkDirP(apiZonesStageDir, 0700); + /* Analyse and process the update. */ ObjectLock olock(updateV1); for (const Dictionary::Pair& kv : updateV1) { @@ -349,13 +423,14 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Utility::MkDirP(currentConfigDir, 0700); Utility::MkDirP(stageConfigDir, 0700); + /* Merge the config information. */ ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = kv.second; if (updateV2) newConfigInfo.UpdateV2 = updateV2->Get(kv.first); - Dictionary::Ptr newConfig = kv.second; + /* Load the current production config details. */ ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); /* Diff the current production configuration with the received configuration. @@ -375,6 +450,17 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D return Empty; } +/** + * Callback for stage config validation. + * When validation was successful, the configuration is copied from + * stage to production and a restart is triggered. + * On failure, there's no restart and this is logged. + * + * @param pr Result of the validation process. + * @param stageConfigDir TODO + * @param currentConfigDir TODO + * @param relativePaths Collected paths which are copied from stage to current. + */ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, const String& stageConfigDir, const String& currentConfigDir, const std::vector& relativePaths) @@ -436,6 +522,14 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, listener->UpdateLastFailedZonesStageValidation(pr.Output); } +/** + * Spawns a new validation process and waits for its output. + * Sets 'System.ZonesStageVarDir' to override the config validation zone dirs with our current stage. + * + * @param stageConfigDir TODO + * @param currentConfigDir TODO + * @param relativePaths Required for later file operations in the callback. + */ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, const std::vector& relativePaths) { @@ -456,6 +550,8 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const } args->Add("--validate"); + + /* Set the ZonesStageDir. This creates our own local chroot without any additional automated zone includes. */ args->Add("--define"); args->Add("System.ZonesStageVarDir=" + GetApiZonesStageDir()); @@ -464,6 +560,12 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths)); } +/** + * Update the structure from the last failed validation output. + * Uses the current timestamp. + * + * @param log The process output from the config validation. + */ void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) { Dictionary::Ptr lastFailedZonesStageValidation = new Dictionary({ @@ -474,6 +576,10 @@ void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) SetLastFailedZonesStageValidation(lastFailedZonesStageValidation); } +/** + * Clear the structure for the last failed reload. + * + */ void ApiListener::ClearLastFailedZonesStageValidation() { SetLastFailedZonesStageValidation(Dictionary::Ptr()); From 043824a6a97153602a05eb78b4f021fcecc9d93b Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 24 Oct 2018 16:52:35 +0200 Subject: [PATCH 18/37] Leave partial deletes as is, this is dealt with stage purge later --- lib/remote/apilistener-filesync.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 79efefe37..4b31673f6 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -186,10 +186,12 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con configChange = true; String path = configDir + "/" + kv.first; - Utility::RemoveDirRecursive(path); + (void) unlink(path.CStr()); } } + /* Consider that one of the paths leaves an empty directory here. Such is not copied from stage to prod and purged then automtically. */ + String tsPath = configDir + "/.timestamp"; if (!Utility::PathExists(tsPath)) { std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); From efc22891787dfa6abf8a8886f8931d4513ac2d37 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 24 Oct 2018 17:06:17 +0200 Subject: [PATCH 19/37] Remove duplicated validation paths in function signatures --- lib/remote/apilistener-filesync.cpp | 33 ++++++++++++----------------- lib/remote/apilistener.hpp | 4 +--- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 4b31673f6..17951a39f 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -446,7 +446,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D /* Spawn a validation process. On success, move the staged configuration * into production and restart. */ - AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths); + AsyncTryActivateZonesStage(relativePaths); } return Empty; @@ -459,20 +459,20 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D * On failure, there's no restart and this is logged. * * @param pr Result of the validation process. - * @param stageConfigDir TODO - * @param currentConfigDir TODO - * @param relativePaths Collected paths which are copied from stage to current. + * @param relativePaths Collected paths including the zone name, which are copied from stage to current directories. */ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, - const String& stageConfigDir, const String& currentConfigDir, const std::vector& relativePaths) { - String logFile = GetApiZonesStageDir() + "/startup.log"; + String apiZonesDir = GetApiZonesDir(); + String apiZonesStageDir = GetApiZonesStageDir(); + + String logFile = apiZonesStageDir + "/startup.log"; std::ofstream fpLog(logFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); fpLog << pr.Output; fpLog.close(); - String statusFile = GetApiZonesStageDir() + "/status"; + String statusFile = apiZonesStageDir + "/status"; std::ofstream fpStatus(statusFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); fpStatus << pr.ExitStatus; fpStatus.close(); @@ -480,9 +480,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* validation went fine, copy stage and reload */ if (pr.ExitStatus == 0) { Log(LogInformation, "ApiListener") - << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, replacing into '" << GetApiZonesDir() << "' and triggering reload."; - - String apiZonesDir = GetApiZonesDir(); + << "Config validation for stage '" << apiZonesStageDir << "' was OK, replacing into '" << apiZonesDir << "' and triggering reload."; /* Purge production before copying stage. */ if (Utility::PathExists(apiZonesDir)) @@ -495,8 +493,8 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Log(LogNotice, "ApiListener") << "Copying file '" << path << "' from config sync staging to production zones directory."; - String stagePath = GetApiZonesStageDir() + path; - String currentPath = GetApiZonesDir() + path; + String stagePath = apiZonesStageDir + path; + String currentPath = apiZonesDir + path; Utility::MkDirP(Utility::DirName(currentPath), 0700); @@ -515,7 +513,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* Error case. */ Log(LogCritical, "ApiListener") - << "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir() + << "Config validation failed for staged cluster config sync in '" << apiZonesStageDir << "'. Aborting. Logs: '" << logFile << "'"; ApiListener::Ptr listener = ApiListener::GetInstance(); @@ -528,12 +526,9 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, * Spawns a new validation process and waits for its output. * Sets 'System.ZonesStageVarDir' to override the config validation zone dirs with our current stage. * - * @param stageConfigDir TODO - * @param currentConfigDir TODO - * @param relativePaths Required for later file operations in the callback. + * @param relativePaths Required for later file operations in the callback. Provides the zone name plus path in a list. */ -void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths) +void ApiListener::AsyncTryActivateZonesStage(const std::vector& relativePaths) { VERIFY(Application::GetArgC() >= 1); @@ -559,7 +554,7 @@ void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const Process::Ptr process = new Process(Process::PrepareCommand(args)); process->SetTimeout(300); - process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths)); + process->Run(std::bind(&TryActivateZonesStageCallback, _1, relativePaths)); } /** diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 59e73ec96..3d1082444 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -182,10 +182,8 @@ private: void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient); static void TryActivateZonesStageCallback(const ProcessResult& pr, - const String& stageConfigDir, const String& currentConfigDir, - const std::vector& relativePaths); - static void AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, const std::vector& relativePaths); + static void AsyncTryActivateZonesStage(const std::vector& relativePaths); void UpdateLastFailedZonesStageValidation(const String& log); void ClearLastFailedZonesStageValidation(); From 9df389a8432d959f8b25e3b6b93404c91a8db26f Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 25 Oct 2018 11:44:49 +0200 Subject: [PATCH 20/37] Improve logging for ignored config updates where we are authoritative for (config master) --- lib/remote/apilistener-filesync.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 17951a39f..a65617318 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -376,9 +376,12 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D */ boost::mutex::scoped_lock lock(m_ConfigSyncStageLock); + String fromEndpointName = origin->FromClient->GetEndpoint()->GetName(); + String fromZoneName = GetFromZoneName(origin->FromZone); + Log(LogInformation, "ApiListener") - << "Applying config update from endpoint '" << origin->FromClient->GetEndpoint()->GetName() - << "' of zone '" << GetFromZoneName(origin->FromZone) << "'."; + << "Applying config update from endpoint '" << fromEndpointName + << "' of zone '" << fromZoneName << "'."; Dictionary::Ptr updateV1 = params->Get("update"); Dictionary::Ptr updateV2 = params->Get("update_v2"); @@ -407,14 +410,16 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D if (!zone) { Log(LogWarning, "ApiListener") - << "Ignoring config update for unknown zone '" << zoneName << "'."; + << "Ignoring config update from endpoint '" << fromEndpointName + << "' for unknown zone '" << zoneName << "'."; continue; } /* Whether we already have configuration in zones.d. */ if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { - Log(LogWarning, "ApiListener") - << "Ignoring config update for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; + Log(LogInformation, "ApiListener") + << "Ignoring config update from endpoint '" << fromEndpointName + << "' for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; continue; } From fcc1799a5de532ffb183aae382b7104fab677b26 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 25 Oct 2018 14:10:30 +0200 Subject: [PATCH 21/37] Split config file sync updates, part I This commit also introduces a playground for checksums, whilst refactoring the code in large parts. --- lib/remote/apilistener-filesync.cpp | 677 ++++++++++++++++------------ lib/remote/apilistener.cpp | 2 +- lib/remote/apilistener.hpp | 14 +- 3 files changed, 389 insertions(+), 304 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index a65617318..cec19e580 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -3,6 +3,8 @@ #include "remote/apilistener.hpp" #include "remote/apifunction.hpp" #include "config/configcompiler.hpp" +#include "base/tlsutility.hpp" +#include "base/json.hpp" #include "base/configtype.hpp" #include "base/logger.hpp" #include "base/convert.hpp" @@ -18,73 +20,312 @@ REGISTER_APIFUNCTION(Update, config, &ApiListener::ConfigUpdateHandler); boost::mutex ApiListener::m_ConfigSyncStageLock; /** - * Read the given file and store it in the config information structure. - * Callback function for Glob(). + * Entrypoint for updating all authoritative configs into var/lib/icinga2/api/zones * - * @param config Reference to the config information object. - * @param path File path. - * @param file Full file name. */ -void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) +void ApiListener::SyncLocalZoneDirs() const { - CONTEXT("Creating config update for file '" + file + "'"); + for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { + try { + SyncLocalZoneDir(zone); + } catch (const std::exception&) { + continue; + } + } +} - Log(LogNotice, "ApiListener") - << "Creating config update for file '" << file << "'."; - - std::ifstream fp(file.CStr(), std::ifstream::binary); - if (!fp) +/** + * Sync a zone directory where we have an authoritative copy (zones.d, etc.) + * + * This function collects the registered zone config dirs from + * the config compiler and reads the file content into the config + * information structure. + * + * Returns early when there are no updates. + * + * @param zone Pointer to the zone object being synced. + */ +void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const +{ + if (!zone) return; - String content((std::istreambuf_iterator(fp)), std::istreambuf_iterator()); + ConfigDirInformation newConfigInfo; + newConfigInfo.UpdateV1 = new Dictionary(); + newConfigInfo.UpdateV2 = new Dictionary(); + newConfigInfo.Checksums = new Dictionary(); - Dictionary::Ptr update; + String zoneName = zone->GetName(); + + /* Load registered zone paths, e.g. '_etc', '_api' and user packages. */ + for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zoneName)) { + ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path); + + /* Config files '*.conf'. */ + { + ObjectLock olock(newConfigPart.UpdateV1); + for (const Dictionary::Pair& kv : newConfigPart.UpdateV1) { + String path = "/" + zf.Tag + kv.first; + newConfigInfo.UpdateV1->Set(path, kv.second); + newConfigInfo.Checksums->Set(path, GetChecksum(kv.second)); + } + } + + /* Meta files. */ + { + ObjectLock olock(newConfigPart.UpdateV2); + for (const Dictionary::Pair& kv : newConfigPart.UpdateV2) { + String path = "/" + zf.Tag + kv.first; + newConfigInfo.UpdateV2->Set(path, kv.second); + newConfigInfo.Checksums->Set(path, GetChecksum(kv.second)); + } + } + } + + int sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength(); + + if (sumUpdates == 0) + return; + + String productionZonesDir = GetApiZonesDir() + zoneName; + + Log(LogInformation, "ApiListener") + << "Copying " << sumUpdates << " zone configuration files for zone '" << zoneName << "' to '" << productionZonesDir << "'."; + + /* Purge files to allow deletion via zones.d. */ + if (Utility::PathExists(productionZonesDir)) + Utility::RemoveDirRecursive(productionZonesDir); + + Utility::MkDirP(productionZonesDir, 0700); + + /* Copy content and add additional meta data. */ + size_t numBytes = 0; + + /* Note: We cannot simply copy directories here. + * + * Zone directories are registered from everywhere and we already + * have read their content into memory with LoadConfigDir(). + */ + Dictionary::Ptr newConfig = MergeConfigUpdate(newConfigInfo); + + { + ObjectLock olock(newConfig); + for (const Dictionary::Pair& kv : newConfig) { + String dst = productionZonesDir + "/" + kv.first; + Utility::MkDirP(Utility::DirName(dst), 0755); + + Log(LogInformation, "ApiListener") + << "Updating configuration file: " << dst; + + String content = kv.second; + std::ofstream fp(dst.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fp << content; + fp.close(); + + numBytes += content.GetLength(); + } + } + + /* Additional metadata. */ + String tsPath = productionZonesDir + "/.timestamp"; + + if (!Utility::PathExists(tsPath)) { + std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp << std::fixed << Utility::GetTime(); + fp.close(); + } + + String authPath = productionZonesDir + "/.authoritative"; + + if (!Utility::PathExists(authPath)) { + std::ofstream fp(authPath.CStr(), std::ofstream::out | std::ostream::trunc); + } + + String checksumsPath = productionZonesDir + "/.checksums"; + + if (Utility::PathExists(checksumsPath)) + (void) unlink(checksumsPath.CStr()); + + std::ofstream fp(checksumsPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp << std::fixed << JsonEncode(newConfigInfo.Checksums); + fp.close(); +} + +/** + * Entrypoint for sending a file based config update to a cluster client. + * This includes security checks for zone relations. + * Loads the zone config files where this client belongs to + * and sends the 'config::Update' JSON-RPC message. + * + * @param aclient Connected JSON-RPC client. + */ +void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) +{ + Endpoint::Ptr endpoint = aclient->GetEndpoint(); + ASSERT(endpoint); + + Zone::Ptr azone = endpoint->GetZone(); + Zone::Ptr lzone = Zone::GetLocalZone(); + + /* don't try to send config updates to our master */ + if (!azone->IsChildOf(lzone)) + return; + + Dictionary::Ptr configUpdateV1 = new Dictionary(); + Dictionary::Ptr configUpdateV2 = new Dictionary(); + Dictionary::Ptr configUpdateChecksums = new Dictionary(); + + String zonesDir = GetApiZonesDir(); + + for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { + String zoneName = zone->GetName(); + String zoneDir = zonesDir + zoneName; + + if (!zone->IsChildOf(azone) && !zone->IsGlobal()) + continue; + + if (!Utility::PathExists(zoneDir)) + continue; + + Log(LogInformation, "ApiListener") + << "Syncing configuration files for " << (zone->IsGlobal() ? "global " : "") + << "zone '" << zoneName << "' to endpoint '" << endpoint->GetName() << "'."; + + ConfigDirInformation config = LoadConfigDir(zoneDir); + + configUpdateV1->Set(zoneName, config.UpdateV1); + configUpdateV2->Set(zoneName, config.UpdateV2); + configUpdateChecksums->Set(zoneName, config.Checksums); + } + + Dictionary::Ptr message = new Dictionary({ + { "jsonrpc", "2.0" }, + { "method", "config::Update" }, + { "params", new Dictionary({ + { "update", configUpdateV1 }, + { "update_v2", configUpdateV2 }, /* Since 2.4.2. */ + { "checksums", configUpdateChecksums } /* Since 2.11.0. */ + }) } + }); + + aclient->SendMessage(message); +} + +/** + * Registered handler when a new config::Update message is received. + * + * Checks destination and permissions first, then analyses the update. + * The newly received configuration is not copied to production immediately, + * but into the staging directory first. + * Last, the async validation and restart is triggered. + * + * @param origin Where this message came from. + * @param params Message parameters including the config updates. + * @returns Empty, required by the interface. + */ +Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) +{ + /* Verify permissions and trust relationship. */ + if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) + return Empty; + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (!listener) { + Log(LogCritical, "ApiListener", "No instance available."); + return Empty; + } + + if (!listener->GetAcceptConfig()) { + Log(LogWarning, "ApiListener") + << "Ignoring config update. '" << listener->GetName() << "' does not accept config."; + return Empty; + } + + /* Only one transaction is allowed, concurrent message handlers need to wait. + * This affects two parent endpoints sending the config in the same moment. + */ + boost::mutex::scoped_lock lock(m_ConfigSyncStageLock); + + String fromEndpointName = origin->FromClient->GetEndpoint()->GetName(); + String fromZoneName = GetFromZoneName(origin->FromZone); + + Log(LogInformation, "ApiListener") + << "Applying config update from endpoint '" << fromEndpointName + << "' of zone '" << fromZoneName << "'."; + + Dictionary::Ptr updateV1 = params->Get("update"); + Dictionary::Ptr updateV2 = params->Get("update_v2"); + + bool configChange = false; + std::vector relativePaths; /* - * 'update' messages contain conf files. 'update_v2' syncs everything else (.timestamp). - * - * **Keep this intact to stay compatible with older clients.** + * We can and must safely purge the staging directory, as the difference is taken between + * runtime production config and newly received configuration. */ - if (Utility::Match("*.conf", file)) - update = config.UpdateV1; - else - update = config.UpdateV2; + String apiZonesStageDir = GetApiZonesStageDir(); - update->Set(file.SubStr(path.GetLength()), content); -} + if (Utility::PathExists(apiZonesStageDir)) + Utility::RemoveDirRecursive(apiZonesStageDir); -/** - * Compatibility helper for merging config update v1 and v2 into a global result. - * - * @param config Config information structure. - * @returns Dictionary which holds the merged information. - */ -Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& config) -{ - Dictionary::Ptr result = new Dictionary(); + Utility::MkDirP(apiZonesStageDir, 0700); - if (config.UpdateV1) - config.UpdateV1->CopyTo(result); + /* Analyse and process the update. */ + ObjectLock olock(updateV1); + for (const Dictionary::Pair& kv : updateV1) { - if (config.UpdateV2) - config.UpdateV2->CopyTo(result); + /* Check for the configured zones. */ + String zoneName = kv.first; + Zone::Ptr zone = Zone::GetByName(zoneName); - return result; -} + if (!zone) { + Log(LogWarning, "ApiListener") + << "Ignoring config update from endpoint '" << fromEndpointName + << "' for unknown zone '" << zoneName << "'."; + continue; + } -/** - * Load the given config dir and read their file content into the config structure. - * - * @param dir Path to the config directory. - * @returns ConfigInformation structure. - */ -ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) -{ - ConfigDirInformation config; - config.UpdateV1 = new Dictionary(); - config.UpdateV2 = new Dictionary(); - Utility::GlobRecursive(dir, "*", std::bind(&ApiListener::ConfigGlobHandler, std::ref(config), dir, _1), GlobFile); - return config; + /* Whether we already have configuration in zones.d. */ + if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { + Log(LogInformation, "ApiListener") + << "Ignoring config update from endpoint '" << fromEndpointName + << "' for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; + continue; + } + + /* Put the received configuration into our stage directory. */ + String currentConfigDir = GetApiZonesDir() + zoneName; + String stageConfigDir = GetApiZonesStageDir() + zoneName; + + Utility::MkDirP(currentConfigDir, 0700); + Utility::MkDirP(stageConfigDir, 0700); + + /* Merge the config information. */ + ConfigDirInformation newConfigInfo; + newConfigInfo.UpdateV1 = kv.second; + + if (updateV2) + newConfigInfo.UpdateV2 = updateV2->Get(kv.first); + + /* Load the current production config details. */ + ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); + + /* Diff the current production configuration with the received configuration. + * If there was a change, collect a signal for later stage validation. + */ + if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) + configChange = true; + } + + if (configChange) { + /* Spawn a validation process. On success, move the staged configuration + * into production and restart. + */ + AsyncTryActivateZonesStage(relativePaths); + } + + return Empty; } /** @@ -210,253 +451,6 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con return configChange; } -/** - * Sync a zone directory where we have an authoritative copy (zones.d, etc.) - * - * This function collects the registered zone config dirs from - * the config compiler and reads the file content into the config - * information structure. - * - * Returns early when there are no updates. - * - * @param zone Pointer to the zone object being synced. - */ -void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const -{ - if (!zone) - return; - - ConfigDirInformation newConfigInfo; - newConfigInfo.UpdateV1 = new Dictionary(); - newConfigInfo.UpdateV2 = new Dictionary(); - - String zoneName = zone->GetName(); - - for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zoneName)) { - ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path); - - { - ObjectLock olock(newConfigPart.UpdateV1); - for (const Dictionary::Pair& kv : newConfigPart.UpdateV1) { - newConfigInfo.UpdateV1->Set("/" + zf.Tag + kv.first, kv.second); - } - } - - { - ObjectLock olock(newConfigPart.UpdateV2); - for (const Dictionary::Pair& kv : newConfigPart.UpdateV2) { - newConfigInfo.UpdateV2->Set("/" + zf.Tag + kv.first, kv.second); - } - } - } - - int sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength(); - - if (sumUpdates == 0) - return; - - String currentDir = Configuration::DataDir + "/api/zones/" + zoneName; - - Log(LogInformation, "ApiListener") - << "Copying " << sumUpdates << " zone configuration files for zone '" << zoneName << "' to '" << currentDir << "'."; - - ConfigDirInformation oldConfigInfo = LoadConfigDir(currentDir); - - /* Purge files to allow deletion via zones.d. */ - Utility::RemoveDirRecursive(currentDir); - Utility::MkDirP(currentDir, 0700); - - std::vector relativePaths; - UpdateConfigDir(oldConfigInfo, newConfigInfo, currentDir, zoneName, relativePaths, true); -} - -/** - * Entrypoint for updating all authoritative configs into var/lib/icinga2/api/zones - * - */ -void ApiListener::SyncZoneDirs() const -{ - for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { - try { - SyncZoneDir(zone); - } catch (const std::exception&) { - continue; - } - } -} - -/** - * Entrypoint for sending a file based config update to a cluster client. - * This includes security checks for zone relations. - * Loads the zone config files where this client belongs to - * and sends the 'config::Update' JSON-RPC message. - * - * @param aclient Connected JSON-RPC client. - */ -void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) -{ - Endpoint::Ptr endpoint = aclient->GetEndpoint(); - ASSERT(endpoint); - - Zone::Ptr azone = endpoint->GetZone(); - Zone::Ptr lzone = Zone::GetLocalZone(); - - /* don't try to send config updates to our master */ - if (!azone->IsChildOf(lzone)) - return; - - Dictionary::Ptr configUpdateV1 = new Dictionary(); - Dictionary::Ptr configUpdateV2 = new Dictionary(); - - String zonesDir = Configuration::DataDir + "/api/zones"; - - for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { - String zoneDir = zonesDir + "/" + zone->GetName(); - - if (!zone->IsChildOf(azone) && !zone->IsGlobal()) - continue; - - if (!Utility::PathExists(zoneDir)) - continue; - - Log(LogInformation, "ApiListener") - << "Syncing configuration files for " << (zone->IsGlobal() ? "global " : "") - << "zone '" << zone->GetName() << "' to endpoint '" << endpoint->GetName() << "'."; - - ConfigDirInformation config = LoadConfigDir(zonesDir + "/" + zone->GetName()); - configUpdateV1->Set(zone->GetName(), config.UpdateV1); - configUpdateV2->Set(zone->GetName(), config.UpdateV2); - } - - Dictionary::Ptr message = new Dictionary({ - { "jsonrpc", "2.0" }, - { "method", "config::Update" }, - { "params", new Dictionary({ - { "update", configUpdateV1 }, - { "update_v2", configUpdateV2 } - }) } - }); - - aclient->SendMessage(message); -} - -/** - * Registered handler when a new config::Update message is received. - * - * Checks destination and permissions first, then analyses the update. - * The newly received configuration is not copied to production immediately, - * but into the staging directory first. - * Last, the async validation and restart is triggered. - * - * @param origin Where this message came from. - * @param params Message parameters including the config updates. - * @returns Empty, required by the interface. - */ -Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) -{ - /* Verify permissions and trust relationship. */ - if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) - return Empty; - - ApiListener::Ptr listener = ApiListener::GetInstance(); - - if (!listener) { - Log(LogCritical, "ApiListener", "No instance available."); - return Empty; - } - - if (!listener->GetAcceptConfig()) { - Log(LogWarning, "ApiListener") - << "Ignoring config update. '" << listener->GetName() << "' does not accept config."; - return Empty; - } - - /* Only one transaction is allowed, concurrent message handlers need to wait. - * This affects two parent endpoints sending the config in the same moment. - */ - boost::mutex::scoped_lock lock(m_ConfigSyncStageLock); - - String fromEndpointName = origin->FromClient->GetEndpoint()->GetName(); - String fromZoneName = GetFromZoneName(origin->FromZone); - - Log(LogInformation, "ApiListener") - << "Applying config update from endpoint '" << fromEndpointName - << "' of zone '" << fromZoneName << "'."; - - Dictionary::Ptr updateV1 = params->Get("update"); - Dictionary::Ptr updateV2 = params->Get("update_v2"); - - bool configChange = false; - std::vector relativePaths; - - /* - * We can and must safely purge the staging directory, as the difference is taken between - * runtime production config and newly received configuration. - */ - String apiZonesStageDir = GetApiZonesStageDir(); - - if (Utility::PathExists(apiZonesStageDir)) - Utility::RemoveDirRecursive(apiZonesStageDir); - - Utility::MkDirP(apiZonesStageDir, 0700); - - /* Analyse and process the update. */ - ObjectLock olock(updateV1); - for (const Dictionary::Pair& kv : updateV1) { - - /* Check for the configured zones. */ - String zoneName = kv.first; - Zone::Ptr zone = Zone::GetByName(zoneName); - - if (!zone) { - Log(LogWarning, "ApiListener") - << "Ignoring config update from endpoint '" << fromEndpointName - << "' for unknown zone '" << zoneName << "'."; - continue; - } - - /* Whether we already have configuration in zones.d. */ - if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { - Log(LogInformation, "ApiListener") - << "Ignoring config update from endpoint '" << fromEndpointName - << "' for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; - continue; - } - - /* Put the received configuration into our stage directory. */ - String currentConfigDir = GetApiZonesDir() + zoneName; - String stageConfigDir = GetApiZonesStageDir() + zoneName; - - Utility::MkDirP(currentConfigDir, 0700); - Utility::MkDirP(stageConfigDir, 0700); - - /* Merge the config information. */ - ConfigDirInformation newConfigInfo; - newConfigInfo.UpdateV1 = kv.second; - - if (updateV2) - newConfigInfo.UpdateV2 = updateV2->Get(kv.first); - - /* Load the current production config details. */ - ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); - - /* Diff the current production configuration with the received configuration. - * If there was a change, collect a signal for later stage validation. - */ - if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) - configChange = true; - } - - if (configChange) { - /* Spawn a validation process. On success, move the staged configuration - * into production and restart. - */ - AsyncTryActivateZonesStage(relativePaths); - } - - return Empty; -} - /** * Callback for stage config validation. * When validation was successful, the configuration is copied from @@ -586,3 +580,90 @@ void ApiListener::ClearLastFailedZonesStageValidation() { SetLastFailedZonesStageValidation(Dictionary::Ptr()); } + +/** + * Generate a config checksum. + * + * @param + */ +String ApiListener::GetChecksum(const String& content) +{ + return SHA256(content); +} + +/** + * Load the given config dir and read their file content into the config structure. + * + * @param dir Path to the config directory. + * @returns ConfigInformation structure. + */ +ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) +{ + ConfigDirInformation config; + config.UpdateV1 = new Dictionary(); + config.UpdateV2 = new Dictionary(); + config.Checksums = new Dictionary(); + + Utility::GlobRecursive(dir, "*", std::bind(&ApiListener::ConfigGlobHandler, std::ref(config), dir, _1), GlobFile); + return config; +} + +/** + * Read the given file and store it in the config information structure. + * Callback function for Glob(). + * + * @param config Reference to the config information object. + * @param path File path. + * @param file Full file name. + */ +void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) +{ + CONTEXT("Creating config update for file '" + file + "'"); + + Log(LogNotice, "ApiListener") + << "Creating config update for file '" << file << "'."; + + std::ifstream fp(file.CStr(), std::ifstream::binary); + if (!fp) + return; + + String content((std::istreambuf_iterator(fp)), std::istreambuf_iterator()); + + Dictionary::Ptr update; + String relativePath = file.SubStr(path.GetLength()); + + /* + * 'update' messages contain conf files. 'update_v2' syncs everything else (.timestamp). + * + * **Keep this intact to stay compatible with older clients.** + */ + if (Utility::Match("*.conf", file)) + update = config.UpdateV1; + else + update = config.UpdateV2; + + update->Set(relativePath, content); + + /* Calculate a checksum for each file (and a global one later). */ + config.Checksums->Set(relativePath, GetChecksum(content)); +} + +/** + * Compatibility helper for merging config update v1 and v2 into a global result. + * + * @param config Config information structure. + * @returns Dictionary which holds the merged information. + */ +Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& config) +{ + Dictionary::Ptr result = new Dictionary(); + + if (config.UpdateV1) + config.UpdateV1->CopyTo(result); + + if (config.UpdateV2) + config.UpdateV2->CopyTo(result); + + return result; +} + diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index b9f8b9780..661a4eaaf 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -242,7 +242,7 @@ void ApiListener::Start(bool runtimeCreated) Log(LogInformation, "ApiListener") << "'" << GetName() << "' started."; - SyncZoneDirs(); + SyncLocalZoneDirs(); ObjectImpl::Start(runtimeCreated); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 3d1082444..f9d92848f 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -32,6 +32,7 @@ struct ConfigDirInformation { Dictionary::Ptr UpdateV1; Dictionary::Ptr UpdateV2; + Dictionary::Ptr Checksums; }; /** @@ -170,21 +171,24 @@ private: /* filesync */ static boost::mutex m_ConfigSyncStageLock; - static ConfigDirInformation LoadConfigDir(const String& dir); + void SyncLocalZoneDirs() const; + void SyncLocalZoneDir(const Zone::Ptr& zone) const; + + void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient); + static Dictionary::Ptr MergeConfigUpdate(const ConfigDirInformation& config); static bool UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative); - void SyncZoneDirs() const; - void SyncZoneDir(const Zone::Ptr& zone) const; - + static ConfigDirInformation LoadConfigDir(const String& dir); static void ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file); - void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient); static void TryActivateZonesStageCallback(const ProcessResult& pr, const std::vector& relativePaths); static void AsyncTryActivateZonesStage(const std::vector& relativePaths); + static String GetChecksum(const String& content); + void UpdateLastFailedZonesStageValidation(const String& log); void ClearLastFailedZonesStageValidation(); From a4b48fc7f4d1db61447d072abf64bc4335a5f48a Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 25 Oct 2018 14:14:19 +0200 Subject: [PATCH 22/37] Update code docs --- lib/remote/apilistener-filesync.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index cec19e580..b6aad6428 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -584,7 +584,8 @@ void ApiListener::ClearLastFailedZonesStageValidation() /** * Generate a config checksum. * - * @param + * @param content String content used for generating the checksum. + * @returns The checksum as string. */ String ApiListener::GetChecksum(const String& content) { From 6105ace50f6331bf2519f21b67ad65c830eed11b Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 25 Oct 2018 14:30:34 +0200 Subject: [PATCH 23/37] Improve variable names in ApiListener::SendConfigUpdate() --- lib/remote/apilistener-filesync.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index b6aad6428..958a6c682 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -164,11 +164,11 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) Endpoint::Ptr endpoint = aclient->GetEndpoint(); ASSERT(endpoint); - Zone::Ptr azone = endpoint->GetZone(); - Zone::Ptr lzone = Zone::GetLocalZone(); + Zone::Ptr clientZone = endpoint->GetZone(); + Zone::Ptr localZone = Zone::GetLocalZone(); /* don't try to send config updates to our master */ - if (!azone->IsChildOf(lzone)) + if (!clientZone->IsChildOf(localZone)) return; Dictionary::Ptr configUpdateV1 = new Dictionary(); @@ -181,9 +181,11 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) String zoneName = zone->GetName(); String zoneDir = zonesDir + zoneName; - if (!zone->IsChildOf(azone) && !zone->IsGlobal()) + /* Only sync child and global zones. */ + if (!zone->IsChildOf(clientZone) && !zone->IsGlobal()) continue; + /* Zone was configured, but there's no configuration directory. */ if (!Utility::PathExists(zoneDir)) continue; From 7a02990ef8a1384599fc74081cf92df5e8546ff1 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 25 Oct 2018 17:47:38 +0200 Subject: [PATCH 24/37] Refactor the client sync, part II (WIP, currently checksums generate an endless loop) --- lib/remote/apilistener-filesync.cpp | 231 +++++++++++++++------------- lib/remote/apilistener.hpp | 3 +- 2 files changed, 122 insertions(+), 112 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 958a6c682..d0c2fcf1b 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -249,6 +249,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D */ boost::mutex::scoped_lock lock(m_ConfigSyncStageLock); + String apiZonesStageDir = GetApiZonesStageDir(); String fromEndpointName = origin->FromClient->GetEndpoint()->GetName(); String fromZoneName = GetFromZoneName(origin->FromZone); @@ -259,6 +260,12 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Dictionary::Ptr updateV1 = params->Get("update"); Dictionary::Ptr updateV2 = params->Get("update_v2"); + /* New since 2.11.0. */ + Dictionary::Ptr checksums; + + if (params->Contains("checksums")) + checksums = params->Get("checksums"); + bool configChange = false; std::vector relativePaths; @@ -266,8 +273,6 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D * We can and must safely purge the staging directory, as the difference is taken between * runtime production config and newly received configuration. */ - String apiZonesStageDir = GetApiZonesStageDir(); - if (Utility::PathExists(apiZonesStageDir)) Utility::RemoveDirRecursive(apiZonesStageDir); @@ -297,106 +302,112 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D } /* Put the received configuration into our stage directory. */ - String currentConfigDir = GetApiZonesDir() + zoneName; - String stageConfigDir = GetApiZonesStageDir() + zoneName; + String productionConfigZoneDir = GetApiZonesDir() + zoneName; + String stageConfigZoneDir = GetApiZonesStageDir() + zoneName; - Utility::MkDirP(currentConfigDir, 0700); - Utility::MkDirP(stageConfigDir, 0700); + Utility::MkDirP(productionConfigZoneDir, 0700); + Utility::MkDirP(stageConfigZoneDir, 0700); /* Merge the config information. */ ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = kv.second; + /* Load metadata. */ if (updateV2) newConfigInfo.UpdateV2 = updateV2->Get(kv.first); + /* Load checksums. */ + if (checksums) + newConfigInfo.Checksums = checksums->Get(kv.first); + /* Load the current production config details. */ - ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); + ConfigDirInformation productionConfigInfo = LoadConfigDir(productionConfigZoneDir); - /* Diff the current production configuration with the received configuration. - * If there was a change, collect a signal for later stage validation. + Dictionary::Ptr productionConfig = MergeConfigUpdate(productionConfigInfo); + Dictionary::Ptr newConfig = MergeConfigUpdate(newConfigInfo); + + /* If we have received 'checksums' via cluster message, go for it. + * Otherwise do the old timestamp dance. */ - if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) - configChange = true; - } + if (checksums) { + /* Calculate and compare the checksums. */ + String productionConfigChecksum = GetGlobalChecksum(productionConfigInfo); + String newConfigChecksum = GetGlobalChecksum(newConfigInfo); - if (configChange) { - /* Spawn a validation process. On success, move the staged configuration - * into production and restart. - */ - AsyncTryActivateZonesStage(relativePaths); - } + Log(LogWarning, "ApiListener") + << "Received configuration for zone '" << zoneName << "' from endpoint '" + << fromEndpointName << "' with checksum '" << newConfigChecksum << "'." + << "Our production configuration has checksum '" << productionConfigChecksum << "'."; - return Empty; -} + /* TODO: Do this earlier in hello-handshakes. */ + if (newConfigChecksum != productionConfigChecksum) + configChange = true; + } else { + /* TODO: Figure out whether we always need to rely on the timestamp flags. */ + double productionTimestamp; -/** - * Diffs the old current configuration with the new configuration - * and copies the collected content. Detects whether a change - * happened, this is used for later restarts. - * - * This generic function is called in two situations: - * - Local zones.d to var/lib/api/zones copy on the master (authoritative: true) - * - Received config update on a cluster node (authoritative: false) - * - * @param oldConfigInfo Config information struct for the current old deployed config. - * @param newConfigInfo Config information struct for the received synced config. - * @param configDir Destination for copying new files (production, or stage dir). - * @param zoneName Currently processed zone, for storing the relative paths for later. - * @param relativePaths Reference which stores all updated config path destinations. - * @param Whether we're authoritative for this config. - * @returns Whether a config change happened. - */ -bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, - const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative) -{ - bool configChange = false; + if (!productionConfig->Contains("/.timestamp")) + productionTimestamp = 0; + else + productionTimestamp = productionConfig->Get("/.timestamp"); - Dictionary::Ptr oldConfig = MergeConfigUpdate(oldConfigInfo); - Dictionary::Ptr newConfig = MergeConfigUpdate(newConfigInfo); + double newTimestamp; - double oldTimestamp; + if (!newConfig->Contains("/.timestamp")) + newTimestamp = Utility::GetTime(); + else + newTimestamp = newConfig->Get("/.timestamp"); - if (!oldConfig->Contains("/.timestamp")) - oldTimestamp = 0; - else - oldTimestamp = oldConfig->Get("/.timestamp"); + /* skip update if our configuration files are more recent */ + if (productionTimestamp >= newTimestamp) { + Log(LogInformation, "ApiListener") + << "Our configuration is more recent than the received configuration update." + << " Ignoring configuration file update for path '" << stageConfigZoneDir << "'. Current timestamp '" + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", productionTimestamp) << "' (" + << std::fixed << std::setprecision(6) << productionTimestamp + << ") >= received timestamp '" + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" + << newTimestamp << ")."; + } else { + configChange = true; + } - double newTimestamp; + /* Keep another hack when there's a timestamp file missing. */ + ObjectLock olock(newConfig); + for (const Dictionary::Pair& kv : newConfig) { + /* This is super expensive with a string content comparison. */ + if (productionConfig->Get(kv.first) != kv.second) { + if (!Utility::Match("*/.timestamp", kv.first)) + configChange = true; + } + } - if (!newConfig->Contains("/.timestamp")) - newTimestamp = Utility::GetTime(); - else - newTimestamp = newConfig->Get("/.timestamp"); + /* Update the .timestamp file. */ + String tsPath = stageConfigZoneDir + "/.timestamp"; + if (!Utility::PathExists(tsPath)) { + std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp << std::fixed << newTimestamp; + fp.close(); + } + } - /* skip update if our configuration files are more recent */ - if (oldTimestamp >= newTimestamp) { - Log(LogNotice, "ApiListener") - << "Our configuration is more recent than the received configuration update." - << " Ignoring configuration file update for path '" << configDir << "'. Current timestamp '" - << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", oldTimestamp) << "' (" - << std::fixed << std::setprecision(6) << oldTimestamp - << ") >= received timestamp '" - << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" - << newTimestamp << ")."; - return false; - } + /* Dump the received configuration for this zone into the stage directory. */ + size_t numBytes = 0; - size_t numBytes = 0; + { + ObjectLock olock(newConfig); + for (const Dictionary::Pair& kv : newConfig) { + /* Ignore same config content. This is an expensive comparison. */ + if (productionConfig->Get(kv.first) == kv.second) + continue; - { - ObjectLock olock(newConfig); - for (const Dictionary::Pair& kv : newConfig) { - if (oldConfig->Get(kv.first) != kv.second) { - if (!Utility::Match("*/.timestamp", kv.first)) - configChange = true; - - /* Store the relative config file path for later. */ + /* Store the relative config file path for later validation and activation. */ relativePaths.push_back(zoneName + "/" + kv.first); - String path = configDir + "/" + kv.first; + String path = stageConfigZoneDir + "/" + kv.first; + Log(LogInformation, "ApiListener") - << "Updating configuration file: " << path; + << "Stage: Updating received configuration file '" << path << "' for zone '" << zoneName << "'."; /* Sync string content only. */ String content = kv.second; @@ -410,47 +421,33 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con numBytes += content.GetLength(); } } - } - /* Log something whether we're authoritative or receing a staged config. */ - Log(LogInformation, "ApiListener") - << "Applying configuration file update for " << (authoritative ? "" : "stage ") - << "path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" - << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" - << std::fixed << std::setprecision(6) << newTimestamp - << "), Current timestamp '" - << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", oldTimestamp) << "' (" - << oldTimestamp << ")."; + Log(LogInformation, "ApiListener") + << "Applying configuration file update for path '" << stageConfigZoneDir << "' (" + << numBytes << " Bytes)."; - /* If the update removes a path, delete it on disk. */ - ObjectLock xlock(oldConfig); - for (const Dictionary::Pair& kv : oldConfig) { - if (!newConfig->Contains(kv.first)) { - configChange = true; + /* If the update removes a path, delete it on disk and signal a config change. */ + { + ObjectLock xlock(productionConfig); + for (const Dictionary::Pair& kv : productionConfig) { + if (!newConfig->Contains(kv.first)) { + configChange = true; - String path = configDir + "/" + kv.first; - (void) unlink(path.CStr()); + String path = stageConfigZoneDir + "/" + kv.first; + (void) unlink(path.CStr()); + } + } } } - /* Consider that one of the paths leaves an empty directory here. Such is not copied from stage to prod and purged then automtically. */ - - String tsPath = configDir + "/.timestamp"; - if (!Utility::PathExists(tsPath)) { - std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); - fp << std::fixed << newTimestamp; - fp.close(); + if (configChange) { + /* Spawn a validation process. On success, move the staged configuration + * into production and restart. + */ + AsyncTryActivateZonesStage(relativePaths); } - if (authoritative) { - String authPath = configDir + "/.authoritative"; - if (!Utility::PathExists(authPath)) { - std::ofstream fp(authPath.CStr(), std::ofstream::out | std::ostream::trunc); - fp.close(); - } - } - - return configChange; + return Empty; } /** @@ -594,11 +591,25 @@ String ApiListener::GetChecksum(const String& content) return SHA256(content); } +String ApiListener::GetGlobalChecksum(const ConfigDirInformation& config) +{ + Dictionary::Ptr checksums = config.Checksums; + + String result; + + ObjectLock olock(checksums); + for (const Dictionary::Pair& kv : checksums) { + result += GetChecksum(kv.second); + } + + return GetChecksum(result); +} + /** * Load the given config dir and read their file content into the config structure. * * @param dir Path to the config directory. - * @returns ConfigInformation structure. + * @returns ConfigDirInformation structure. */ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) { diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index f9d92848f..393933ace 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -177,8 +177,6 @@ private: void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient); static Dictionary::Ptr MergeConfigUpdate(const ConfigDirInformation& config); - static bool UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, - const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative); static ConfigDirInformation LoadConfigDir(const String& dir); static void ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file); @@ -188,6 +186,7 @@ private: static void AsyncTryActivateZonesStage(const std::vector& relativePaths); static String GetChecksum(const String& content); + static String GetGlobalChecksum(const ConfigDirInformation& config); void UpdateLastFailedZonesStageValidation(const String& log); void ClearLastFailedZonesStageValidation(); From c230e503e6eec2479c2a441573ef2075b0d776ac Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 26 Oct 2018 14:20:14 +0200 Subject: [PATCH 25/37] Fix global checksum calculation --- lib/remote/apilistener-filesync.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index d0c2fcf1b..95c7df0dd 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -599,7 +599,22 @@ String ApiListener::GetGlobalChecksum(const ConfigDirInformation& config) ObjectLock olock(checksums); for (const Dictionary::Pair& kv : checksums) { - result += GetChecksum(kv.second); + String path = kv.first; + String checksum = kv.second; + + /* Only use configuration files for checksum calculation. */ + //if (!Utility::Match("*.conf", path)) + // continue; + + /* Ignore internal files, especially .timestamp and .checksums. + * If we don't, this results in "always change" restart loops. + */ + if (Utility::Match("/.*", path)) + continue; + + Log(LogCritical, "ApiListener") + << "Adding checksum for " << kv.first << ": " << kv.second; + result += kv.second; } return GetChecksum(result); From a6ddef17d9661aedc876913bb199ac7ce30aa141 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 26 Oct 2018 14:20:27 +0200 Subject: [PATCH 26/37] Enhace logging when config change yes/no will trigger further reload actions --- lib/remote/apilistener-filesync.cpp | 33 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 95c7df0dd..206eac61b 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -257,7 +257,9 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D << "Applying config update from endpoint '" << fromEndpointName << "' of zone '" << fromZoneName << "'."; + /* Config files. */ Dictionary::Ptr updateV1 = params->Get("update"); + /* Meta data files: .timestamp, etc. */ Dictionary::Ptr updateV2 = params->Get("update_v2"); /* New since 2.11.0. */ @@ -267,11 +269,15 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D checksums = params->Get("checksums"); bool configChange = false; + + /* Keep track of the relative config paths for later validation and copying. */ std::vector relativePaths; /* * We can and must safely purge the staging directory, as the difference is taken between * runtime production config and newly received configuration. + * This is needed to not mix deleted/changed content between received and stage + * config. */ if (Utility::PathExists(apiZonesStageDir)) Utility::RemoveDirRecursive(apiZonesStageDir); @@ -316,7 +322,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D if (updateV2) newConfigInfo.UpdateV2 = updateV2->Get(kv.first); - /* Load checksums. */ + /* Load checksums. New since 2.11. */ if (checksums) newConfigInfo.Checksums = checksums->Get(kv.first); @@ -327,23 +333,23 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Dictionary::Ptr newConfig = MergeConfigUpdate(newConfigInfo); /* If we have received 'checksums' via cluster message, go for it. - * Otherwise do the old timestamp dance. + * Otherwise do the old timestamp dance for versions < 2.11. */ if (checksums) { /* Calculate and compare the checksums. */ String productionConfigChecksum = GetGlobalChecksum(productionConfigInfo); String newConfigChecksum = GetGlobalChecksum(newConfigInfo); - Log(LogWarning, "ApiListener") + Log(LogInformation, "ApiListener") << "Received configuration for zone '" << zoneName << "' from endpoint '" << fromEndpointName << "' with checksum '" << newConfigChecksum << "'." - << "Our production configuration has checksum '" << productionConfigChecksum << "'."; + << " Our production configuration has checksum '" << productionConfigChecksum << "'."; /* TODO: Do this earlier in hello-handshakes. */ if (newConfigChecksum != productionConfigChecksum) configChange = true; } else { - /* TODO: Figure out whether we always need to rely on the timestamp flags. */ + /* TODO: Figure out whether we always need to rely on the timestamp flags when there are checksums involved. */ double productionTimestamp; if (!productionConfig->Contains("/.timestamp")) @@ -440,11 +446,22 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D } } + /* + * We have processed all configuration files and stored them in the staging directory. + * We need to store them locally for later analysis. A config change means + * that we will validate the configuration in a separate process sandbox, + * and only copy the configuration to production when everything is ok. + * A successful validation also triggers the final restart. + */ if (configChange) { - /* Spawn a validation process. On success, move the staged configuration - * into production and restart. - */ + Log(LogInformation, "ApiListener") + << "Received configuration from endpoint '" << fromEndpointName + << "' is different to production, triggering validation and reload."; AsyncTryActivateZonesStage(relativePaths); + } else { + Log(LogInformation, "ApiListener") + << "Received configuration from endpoint '" << fromEndpointName + << "' is equal to production, not triggering reload."; } return Empty; From f92f6f7f8cac858cbbd59d757f343a3c68f90596 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 26 Oct 2018 16:05:21 +0200 Subject: [PATCH 27/37] Improve checksum checks for each file content --- lib/remote/apilistener-filesync.cpp | 38 ++++++++++++++++------------- lib/remote/apilistener.hpp | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 206eac61b..12d718a2d 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -336,17 +336,12 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D * Otherwise do the old timestamp dance for versions < 2.11. */ if (checksums) { - /* Calculate and compare the checksums. */ - String productionConfigChecksum = GetGlobalChecksum(productionConfigInfo); - String newConfigChecksum = GetGlobalChecksum(newConfigInfo); - Log(LogInformation, "ApiListener") << "Received configuration for zone '" << zoneName << "' from endpoint '" - << fromEndpointName << "' with checksum '" << newConfigChecksum << "'." - << " Our production configuration has checksum '" << productionConfigChecksum << "'."; + << fromEndpointName << "'. Comparing the checksums."; /* TODO: Do this earlier in hello-handshakes. */ - if (newConfigChecksum != productionConfigChecksum) + if (CheckConfigChange(productionConfigInfo, newConfigInfo)) configChange = true; } else { /* TODO: Figure out whether we always need to rely on the timestamp flags when there are checksums involved. */ @@ -608,17 +603,23 @@ String ApiListener::GetChecksum(const String& content) return SHA256(content); } -String ApiListener::GetGlobalChecksum(const ConfigDirInformation& config) +bool ApiListener::CheckConfigChange(const ConfigDirInformation& oldConfig, const ConfigDirInformation& newConfig) { - Dictionary::Ptr checksums = config.Checksums; + Dictionary::Ptr oldChecksums = oldConfig.Checksums; + Dictionary::Ptr newChecksums = newConfig.Checksums; - String result; + Log(LogCritical, "Comparing old: '") + << JsonEncode(oldChecksums) + << "' to new '" << JsonEncode(newChecksums) << "'."; - ObjectLock olock(checksums); - for (const Dictionary::Pair& kv : checksums) { + /* Different length means that either one or the other side added or removed something. */ + if (oldChecksums->GetLength() != newChecksums->GetLength()) + return true; + + /* Both dictionaries have an equal size. */ + ObjectLock olock(oldChecksums); + for (const Dictionary::Pair& kv : oldChecksums) { String path = kv.first; - String checksum = kv.second; - /* Only use configuration files for checksum calculation. */ //if (!Utility::Match("*.conf", path)) // continue; @@ -630,11 +631,14 @@ String ApiListener::GetGlobalChecksum(const ConfigDirInformation& config) continue; Log(LogCritical, "ApiListener") - << "Adding checksum for " << kv.first << ": " << kv.second; - result += kv.second; + << "Checking " << path << " for checksum: " << kv.second; + + /* Check whether our key exists in the new checksums, and they have an equal value. */ + if (newChecksums->Get(path) != kv.second) + return true; } - return GetChecksum(result); + return false; } /** diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 393933ace..0f552e984 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -186,7 +186,7 @@ private: static void AsyncTryActivateZonesStage(const std::vector& relativePaths); static String GetChecksum(const String& content); - static String GetGlobalChecksum(const ConfigDirInformation& config); + static bool CheckConfigChange(const ConfigDirInformation& oldConfig, const ConfigDirInformation& newConfig); void UpdateLastFailedZonesStageValidation(const String& log); void ClearLastFailedZonesStageValidation(); From 4c6150b254b24c11a9a08fabaf28b6817c8263f2 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 26 Oct 2018 16:29:46 +0200 Subject: [PATCH 28/37] Improve checksum logic and logging --- lib/remote/apilistener-filesync.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 12d718a2d..845be35ee 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -398,13 +398,14 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D { ObjectLock olock(newConfig); for (const Dictionary::Pair& kv : newConfig) { + /* Store the relative config file path for later validation and activation. + * IMPORTANT: Store this prior to any filters. */ + relativePaths.push_back(zoneName + "/" + kv.first); + /* Ignore same config content. This is an expensive comparison. */ if (productionConfig->Get(kv.first) == kv.second) continue; - /* Store the relative config file path for later validation and activation. */ - relativePaths.push_back(zoneName + "/" + kv.first); - String path = stageConfigZoneDir + "/" + kv.first; Log(LogInformation, "ApiListener") @@ -608,9 +609,11 @@ bool ApiListener::CheckConfigChange(const ConfigDirInformation& oldConfig, const Dictionary::Ptr oldChecksums = oldConfig.Checksums; Dictionary::Ptr newChecksums = newConfig.Checksums; - Log(LogCritical, "Comparing old: '") + Log(LogCritical, "ApiListener") + << "Comparing old (" << oldChecksums->GetLength() << "): '" << JsonEncode(oldChecksums) - << "' to new '" << JsonEncode(newChecksums) << "'."; + << "' to new (" << newChecksums->GetLength() << "): '" + << JsonEncode(newChecksums) << "'."; /* Different length means that either one or the other side added or removed something. */ if (oldChecksums->GetLength() != newChecksums->GetLength()) @@ -620,6 +623,7 @@ bool ApiListener::CheckConfigChange(const ConfigDirInformation& oldConfig, const ObjectLock olock(oldChecksums); for (const Dictionary::Pair& kv : oldChecksums) { String path = kv.first; + String oldChecksum = kv.second; /* Only use configuration files for checksum calculation. */ //if (!Utility::Match("*.conf", path)) // continue; @@ -631,11 +635,17 @@ bool ApiListener::CheckConfigChange(const ConfigDirInformation& oldConfig, const continue; Log(LogCritical, "ApiListener") - << "Checking " << path << " for checksum: " << kv.second; + << "Checking " << path << " for checksum: " << oldChecksum; /* Check whether our key exists in the new checksums, and they have an equal value. */ - if (newChecksums->Get(path) != kv.second) + String newChecksum = newChecksums->Get(path); + + if (newChecksums->Get(path) != kv.second) { + Log(LogCritical, "ApiListener") + << "Path '" << path << "' doesn't match old checksum '" + << newChecksum << "' with new checksum '" << oldChecksum << "'."; return true; + } } return false; From b3b7abdfe884d07a05e0457717f9fe1aa886c28d Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 26 Oct 2018 16:54:55 +0200 Subject: [PATCH 29/37] Spam the log with config file copies from stage to prod --- lib/remote/apilistener-filesync.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 845be35ee..e9a54a88f 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -501,7 +501,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, /* Copy all synced configuration files from stage to production. */ for (const String& path : relativePaths) { - Log(LogNotice, "ApiListener") + Log(LogInformation, "ApiListener") << "Copying file '" << path << "' from config sync staging to production zones directory."; String stagePath = apiZonesStageDir + path; From af8624dcf1c4bf5e0bcabaad251e8e8f3c2c2a64 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 13 May 2019 11:22:55 +0200 Subject: [PATCH 30/37] Apply ReloadTimeout for 2.11 --- lib/remote/apilistener-filesync.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index e9a54a88f..bd592afaf 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -8,6 +8,7 @@ #include "base/configtype.hpp" #include "base/logger.hpp" #include "base/convert.hpp" +#include "base/application.hpp" #include "base/exception.hpp" #include "base/utility.hpp" #include @@ -564,7 +565,7 @@ void ApiListener::AsyncTryActivateZonesStage(const std::vector& relative args->Add("System.ZonesStageVarDir=" + GetApiZonesStageDir()); Process::Ptr process = new Process(Process::PrepareCommand(args)); - process->SetTimeout(300); + process->SetTimeout(Application::GetReloadTimeout()); process->Run(std::bind(&TryActivateZonesStageCallback, _1, relativePaths)); } From 6add9f9ecb3280c711105e5792b9b133cfa14b53 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 13 May 2019 11:26:39 +0200 Subject: [PATCH 31/37] Avoid concurrent cluster config sync transactions fixes #6660 --- lib/remote/apilistener-filesync.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index bd592afaf..e39945a36 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -217,7 +217,7 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) /** * Registered handler when a new config::Update message is received. * - * Checks destination and permissions first, then analyses the update. + * Checks destination and permissions first, locks the transaction and analyses the update. * The newly received configuration is not copied to production immediately, * but into the staging directory first. * Last, the async validation and restart is triggered. From 0aa6f1a3b364902acc0d2f8bf51c9004edac8be0 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 13 May 2019 11:27:49 +0200 Subject: [PATCH 32/37] Use boost::filesystem & Utility classes for file IO --- lib/remote/apilistener-filesync.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index e39945a36..531748737 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -83,7 +83,7 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const } } - int sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength(); + size_t sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength(); if (sumUpdates == 0) return; @@ -140,16 +140,21 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const if (!Utility::PathExists(authPath)) { std::ofstream fp(authPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp.close(); } String checksumsPath = productionZonesDir + "/.checksums"; if (Utility::PathExists(checksumsPath)) - (void) unlink(checksumsPath.CStr()); + Utility::Remove(checksumsPath); std::ofstream fp(checksumsPath.CStr(), std::ofstream::out | std::ostream::trunc); fp << std::fixed << JsonEncode(newConfigInfo.Checksums); fp.close(); + + Log(LogNotice, "ApiListener") + << "Updated meta data for cluster config sync. Checksum: '" << checksumsPath + << "', timestamp: '" << tsPath << "', auth: '" << authPath << "'."; } /** @@ -174,7 +179,7 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) Dictionary::Ptr configUpdateV1 = new Dictionary(); Dictionary::Ptr configUpdateV2 = new Dictionary(); - Dictionary::Ptr configUpdateChecksums = new Dictionary(); + Dictionary::Ptr configUpdateChecksums = new Dictionary(); /* new since 2.11 */ String zonesDir = GetApiZonesDir(); @@ -198,7 +203,7 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) configUpdateV1->Set(zoneName, config.UpdateV1); configUpdateV2->Set(zoneName, config.UpdateV2); - configUpdateChecksums->Set(zoneName, config.Checksums); + configUpdateChecksums->Set(zoneName, config.Checksums); /* new since 2.11 */ } Dictionary::Ptr message = new Dictionary({ @@ -437,7 +442,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D configChange = true; String path = stageConfigZoneDir + "/" + kv.first; - (void) unlink(path.CStr()); + Utility::Remove(path); } } } From 3852c51c9f96b8e65077c70b392458a298dabadf Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 6 Jun 2019 17:45:43 +0200 Subject: [PATCH 33/37] Cluster sync: Don't load/sync the .authoritative config file marker This would influence everything else, and it isn't needed anywhere but the master instance (zones.d -> var-zones). --- lib/remote/apilistener-filesync.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 531748737..6279be5cc 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -684,6 +684,10 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) */ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) { + /* Avoid loading the authoritative marker for syncs. */ + if (Utility::BaseName(file) == ".authoritative") + return; + CONTEXT("Creating config update for file '" + file + "'"); Log(LogNotice, "ApiListener") @@ -710,7 +714,10 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& update->Set(relativePath, content); - /* Calculate a checksum for each file (and a global one later). */ + /* Calculate a checksum for each file (and a global one later). + * + * IMPORTANT: Ignore the .authoritative file, this will not be synced. + * */ config.Checksums->Set(relativePath, GetChecksum(content)); } From 577e42e1373237a7336d218119b504a4d52563cc Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 7 Jun 2019 11:32:27 +0200 Subject: [PATCH 34/37] Quality: Comments and logs in cluster config sync --- lib/remote/apilistener-filesync.cpp | 171 +++++++++++++++++----------- 1 file changed, 107 insertions(+), 64 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 6279be5cc..f1ed77576 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -21,8 +21,8 @@ REGISTER_APIFUNCTION(Update, config, &ApiListener::ConfigUpdateHandler); boost::mutex ApiListener::m_ConfigSyncStageLock; /** - * Entrypoint for updating all authoritative configs into var/lib/icinga2/api/zones - * + * Entrypoint for updating all authoritative configs from /etc/zones.d, packages, etc. + * into var/lib/icinga2/api/zones */ void ApiListener::SyncLocalZoneDirs() const { @@ -36,7 +36,7 @@ void ApiListener::SyncLocalZoneDirs() const } /** - * Sync a zone directory where we have an authoritative copy (zones.d, etc.) + * Sync a zone directory where we have an authoritative copy (zones.d, packages, etc.) * * This function collects the registered zone config dirs from * the config compiler and reads the file content into the config @@ -58,25 +58,27 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const String zoneName = zone->GetName(); - /* Load registered zone paths, e.g. '_etc', '_api' and user packages. */ + // Load registered zone paths, e.g. '_etc', '_api' and user packages. for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zoneName)) { ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path); - /* Config files '*.conf'. */ + // Config files '*.conf'. { ObjectLock olock(newConfigPart.UpdateV1); for (const Dictionary::Pair& kv : newConfigPart.UpdateV1) { String path = "/" + zf.Tag + kv.first; + newConfigInfo.UpdateV1->Set(path, kv.second); newConfigInfo.Checksums->Set(path, GetChecksum(kv.second)); } } - /* Meta files. */ + // Meta files. { ObjectLock olock(newConfigPart.UpdateV2); for (const Dictionary::Pair& kv : newConfigPart.UpdateV2) { String path = "/" + zf.Tag + kv.first; + newConfigInfo.UpdateV2->Set(path, kv.second); newConfigInfo.Checksums->Set(path, GetChecksum(kv.second)); } @@ -85,6 +87,7 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const size_t sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength(); + // Return early if there are no updates. if (sumUpdates == 0) return; @@ -93,13 +96,13 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const Log(LogInformation, "ApiListener") << "Copying " << sumUpdates << " zone configuration files for zone '" << zoneName << "' to '" << productionZonesDir << "'."; - /* Purge files to allow deletion via zones.d. */ + // Purge files to allow deletion via zones.d. if (Utility::PathExists(productionZonesDir)) Utility::RemoveDirRecursive(productionZonesDir); Utility::MkDirP(productionZonesDir, 0700); - /* Copy content and add additional meta data. */ + // Copy content and add additional meta data. size_t numBytes = 0; /* Note: We cannot simply copy directories here. @@ -111,15 +114,19 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const { ObjectLock olock(newConfig); + for (const Dictionary::Pair& kv : newConfig) { String dst = productionZonesDir + "/" + kv.first; + Utility::MkDirP(Utility::DirName(dst), 0755); Log(LogInformation, "ApiListener") << "Updating configuration file: " << dst; String content = kv.second; + std::ofstream fp(dst.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fp << content; fp.close(); @@ -127,11 +134,12 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const } } - /* Additional metadata. */ + // Additional metadata. String tsPath = productionZonesDir + "/.timestamp"; if (!Utility::PathExists(tsPath)) { std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp << std::fixed << Utility::GetTime(); fp.close(); } @@ -143,12 +151,14 @@ void ApiListener::SyncLocalZoneDir(const Zone::Ptr& zone) const fp.close(); } + // Checksums. String checksumsPath = productionZonesDir + "/.checksums"; if (Utility::PathExists(checksumsPath)) Utility::Remove(checksumsPath); std::ofstream fp(checksumsPath.CStr(), std::ofstream::out | std::ostream::trunc); + fp << std::fixed << JsonEncode(newConfigInfo.Checksums); fp.close(); @@ -173,13 +183,13 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) Zone::Ptr clientZone = endpoint->GetZone(); Zone::Ptr localZone = Zone::GetLocalZone(); - /* don't try to send config updates to our master */ + // Don't send config updates to parent zones if (!clientZone->IsChildOf(localZone)) return; Dictionary::Ptr configUpdateV1 = new Dictionary(); Dictionary::Ptr configUpdateV2 = new Dictionary(); - Dictionary::Ptr configUpdateChecksums = new Dictionary(); /* new since 2.11 */ + Dictionary::Ptr configUpdateChecksums = new Dictionary(); // new since 2.11 String zonesDir = GetApiZonesDir(); @@ -187,11 +197,11 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) String zoneName = zone->GetName(); String zoneDir = zonesDir + zoneName; - /* Only sync child and global zones. */ + // Only sync child and global zones. if (!zone->IsChildOf(clientZone) && !zone->IsGlobal()) continue; - /* Zone was configured, but there's no configuration directory. */ + // Zone was configured, but there's no configuration directory. if (!Utility::PathExists(zoneDir)) continue; @@ -203,7 +213,7 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) configUpdateV1->Set(zoneName, config.UpdateV1); configUpdateV2->Set(zoneName, config.UpdateV2); - configUpdateChecksums->Set(zoneName, config.Checksums); /* new since 2.11 */ + configUpdateChecksums->Set(zoneName, config.Checksums); // new since 2.11 } Dictionary::Ptr message = new Dictionary({ @@ -211,8 +221,8 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) { "method", "config::Update" }, { "params", new Dictionary({ { "update", configUpdateV1 }, - { "update_v2", configUpdateV2 }, /* Since 2.4.2. */ - { "checksums", configUpdateChecksums } /* Since 2.11.0. */ + { "update_v2", configUpdateV2 }, // Since 2.4.2. + { "checksums", configUpdateChecksums } // Since 2.11.0. }) } }); @@ -233,7 +243,7 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) */ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { - /* Verify permissions and trust relationship. */ + // Verify permissions and trust relationship. if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) return Empty; @@ -263,12 +273,12 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D << "Applying config update from endpoint '" << fromEndpointName << "' of zone '" << fromZoneName << "'."; - /* Config files. */ + // Config files. Dictionary::Ptr updateV1 = params->Get("update"); - /* Meta data files: .timestamp, etc. */ + // Meta data files: .timestamp, etc. Dictionary::Ptr updateV2 = params->Get("update_v2"); - /* New since 2.11.0. */ + // New since 2.11.0. Dictionary::Ptr checksums; if (params->Contains("checksums")) @@ -276,7 +286,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D bool configChange = false; - /* Keep track of the relative config paths for later validation and copying. */ + // Keep track of the relative config paths for later validation and copying. TODO: Find a better algorithm. std::vector relativePaths; /* @@ -290,11 +300,12 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Utility::MkDirP(apiZonesStageDir, 0700); - /* Analyse and process the update. */ + // Analyse and process the update. ObjectLock olock(updateV1); + for (const Dictionary::Pair& kv : updateV1) { - /* Check for the configured zones. */ + // Check for the configured zones. String zoneName = kv.first; Zone::Ptr zone = Zone::GetByName(zoneName); @@ -302,39 +313,42 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Log(LogWarning, "ApiListener") << "Ignoring config update from endpoint '" << fromEndpointName << "' for unknown zone '" << zoneName << "'."; + continue; } - /* Whether we already have configuration in zones.d. */ + // Ignore updates where we have an authoritive copy in etc/zones.d, packages, etc. if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { Log(LogInformation, "ApiListener") << "Ignoring config update from endpoint '" << fromEndpointName << "' for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; + continue; } - /* Put the received configuration into our stage directory. */ + // Put the received configuration into our stage directory. String productionConfigZoneDir = GetApiZonesDir() + zoneName; String stageConfigZoneDir = GetApiZonesStageDir() + zoneName; Utility::MkDirP(productionConfigZoneDir, 0700); Utility::MkDirP(stageConfigZoneDir, 0700); - /* Merge the config information. */ + // Merge the config information. ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = kv.second; - /* Load metadata. */ + // Load metadata. if (updateV2) newConfigInfo.UpdateV2 = updateV2->Get(kv.first); - /* Load checksums. New since 2.11. */ + // Load checksums. New since 2.11. if (checksums) newConfigInfo.Checksums = checksums->Get(kv.first); - /* Load the current production config details. */ + // Load the current production config details. ConfigDirInformation productionConfigInfo = LoadConfigDir(productionConfigZoneDir); + // Merge updateV1 and updateV2 Dictionary::Ptr productionConfig = MergeConfigUpdate(productionConfigInfo); Dictionary::Ptr newConfig = MergeConfigUpdate(newConfigInfo); @@ -346,11 +360,21 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D << "Received configuration for zone '" << zoneName << "' from endpoint '" << fromEndpointName << "'. Comparing the checksums."; - /* TODO: Do this earlier in hello-handshakes. */ + // TODO: Do this earlier in hello-handshakes? if (CheckConfigChange(productionConfigInfo, newConfigInfo)) configChange = true; + } else { - /* TODO: Figure out whether we always need to rely on the timestamp flags when there are checksums involved. */ + /* Fallback to timestamp handling when the parent endpoint didn't send checks. + * This can happen when the satellite is 2.11 and the master is 2.10. + * + * TODO: Deprecate and remove this behaviour in 2.13+. + */ + + Log(LogWarning, "ApiListener") + << "Received configuration update without checksums from parent endpoint " + << fromEndpointName << ". This behaviour is deprecated. Please upgrade the parent endpoint to 2.11+"; + double productionTimestamp; if (!productionConfig->Contains("/.timestamp")) @@ -365,8 +389,9 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D else newTimestamp = newConfig->Get("/.timestamp"); - /* skip update if our configuration files are more recent */ + // Skip update if our configuration files are more recent. if (productionTimestamp >= newTimestamp) { + Log(LogInformation, "ApiListener") << "Our configuration is more recent than the received configuration update." << " Ignoring configuration file update for path '" << stageConfigZoneDir << "'. Current timestamp '" @@ -375,21 +400,26 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D << ") >= received timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" << newTimestamp << ")."; + } else { configChange = true; } - /* Keep another hack when there's a timestamp file missing. */ - ObjectLock olock(newConfig); - for (const Dictionary::Pair& kv : newConfig) { - /* This is super expensive with a string content comparison. */ - if (productionConfig->Get(kv.first) != kv.second) { - if (!Utility::Match("*/.timestamp", kv.first)) - configChange = true; + // Keep another hack when there's a timestamp file missing. + { + ObjectLock olock(newConfig); + + for (const Dictionary::Pair &kv : newConfig) { + + // This is super expensive with a string content comparison. + if (productionConfig->Get(kv.first) != kv.second) { + if (!Utility::Match("*/.timestamp", kv.first)) + configChange = true; + } } } - /* Update the .timestamp file. */ + // Update the .timestamp file. String tsPath = stageConfigZoneDir + "/.timestamp"; if (!Utility::PathExists(tsPath)) { std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); @@ -398,17 +428,20 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D } } - /* Dump the received configuration for this zone into the stage directory. */ + // Dump the received configuration for this zone into the stage directory. size_t numBytes = 0; { ObjectLock olock(newConfig); + for (const Dictionary::Pair& kv : newConfig) { + /* Store the relative config file path for later validation and activation. - * IMPORTANT: Store this prior to any filters. */ + * IMPORTANT: Store this prior to any filters. + * */ relativePaths.push_back(zoneName + "/" + kv.first); - /* Ignore same config content. This is an expensive comparison. */ + // Ignore same config content. This is an expensive comparison. if (productionConfig->Get(kv.first) == kv.second) continue; @@ -417,11 +450,13 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Log(LogInformation, "ApiListener") << "Stage: Updating received configuration file '" << path << "' for zone '" << zoneName << "'."; - /* Sync string content only. */ + // Sync string content only. String content = kv.second; - /* Generate a directory tree (zones/1/2/3 might not exist yet). */ + // Generate a directory tree (zones/1/2/3 might not exist yet). Utility::MkDirP(Utility::DirName(path), 0755); + + // Write the content to file. std::ofstream fp(path.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); fp << content; fp.close(); @@ -434,9 +469,10 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D << "Applying configuration file update for path '" << stageConfigZoneDir << "' (" << numBytes << " Bytes)."; - /* If the update removes a path, delete it on disk and signal a config change. */ + // If the update removes a path, delete it on disk and signal a config change. { ObjectLock xlock(productionConfig); + for (const Dictionary::Pair& kv : productionConfig) { if (!newConfig->Contains(kv.first)) { configChange = true; @@ -450,9 +486,11 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D /* * We have processed all configuration files and stored them in the staging directory. + * * We need to store them locally for later analysis. A config change means * that we will validate the configuration in a separate process sandbox, * and only copy the configuration to production when everything is ok. + * * A successful validation also triggers the final restart. */ if (configChange) { @@ -494,18 +532,18 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, fpStatus << pr.ExitStatus; fpStatus.close(); - /* validation went fine, copy stage and reload */ + // Validation went fine, copy stage and reload. if (pr.ExitStatus == 0) { Log(LogInformation, "ApiListener") << "Config validation for stage '" << apiZonesStageDir << "' was OK, replacing into '" << apiZonesDir << "' and triggering reload."; - /* Purge production before copying stage. */ + // Purge production before copying stage. if (Utility::PathExists(apiZonesDir)) Utility::RemoveDirRecursive(apiZonesDir); Utility::MkDirP(apiZonesDir, 0700); - /* Copy all synced configuration files from stage to production. */ + // Copy all synced configuration files from stage to production. for (const String& path : relativePaths) { Log(LogInformation, "ApiListener") << "Copying file '" << path << "' from config sync staging to production zones directory."; @@ -518,6 +556,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Utility::CopyFile(stagePath, currentPath); } + // Clear any failed deployment before ApiListener::Ptr listener = ApiListener::GetInstance(); if (listener) @@ -525,10 +564,11 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, Application::RequestRestart(); + // All good, return early. return; } - /* Error case. */ + // Error case. Log(LogCritical, "ApiListener") << "Config validation failed for staged cluster config sync in '" << apiZonesStageDir << "'. Aborting. Logs: '" << logFile << "'"; @@ -565,7 +605,7 @@ void ApiListener::AsyncTryActivateZonesStage(const std::vector& relative args->Add("--validate"); - /* Set the ZonesStageDir. This creates our own local chroot without any additional automated zone includes. */ + // Set the ZonesStageDir. This creates our own local chroot without any additional automated zone includes. args->Add("--define"); args->Add("System.ZonesStageVarDir=" + GetApiZonesStageDir()); @@ -615,39 +655,43 @@ bool ApiListener::CheckConfigChange(const ConfigDirInformation& oldConfig, const Dictionary::Ptr oldChecksums = oldConfig.Checksums; Dictionary::Ptr newChecksums = newConfig.Checksums; - Log(LogCritical, "ApiListener") - << "Comparing old (" << oldChecksums->GetLength() << "): '" + // TODO: Figure out whether normal users need this for debugging. + Log(LogDebug, "ApiListener") + << "Checking for config change between stage and production. Old (" << oldChecksums->GetLength() << "): '" << JsonEncode(oldChecksums) - << "' to new (" << newChecksums->GetLength() << "): '" + << "' vs. new (" << newChecksums->GetLength() << "): '" << JsonEncode(newChecksums) << "'."; - /* Different length means that either one or the other side added or removed something. */ + // Different length means that either one or the other side added or removed something. */ if (oldChecksums->GetLength() != newChecksums->GetLength()) return true; - /* Both dictionaries have an equal size. */ + // Both dictionaries have an equal size. ObjectLock olock(oldChecksums); + for (const Dictionary::Pair& kv : oldChecksums) { String path = kv.first; String oldChecksum = kv.second; - /* Only use configuration files for checksum calculation. */ + + // TODO: Figure out if config changes only apply to '.conf'. Leaving this open for other config files. //if (!Utility::Match("*.conf", path)) // continue; /* Ignore internal files, especially .timestamp and .checksums. + * * If we don't, this results in "always change" restart loops. */ if (Utility::Match("/.*", path)) continue; - Log(LogCritical, "ApiListener") + Log(LogDebug, "ApiListener") << "Checking " << path << " for checksum: " << oldChecksum; - /* Check whether our key exists in the new checksums, and they have an equal value. */ + // Check whether our key exists in the new checksums, and they have an equal value. String newChecksum = newChecksums->Get(path); if (newChecksums->Get(path) != kv.second) { - Log(LogCritical, "ApiListener") + Log(LogDebug, "ApiListener") << "Path '" << path << "' doesn't match old checksum '" << newChecksum << "' with new checksum '" << oldChecksum << "'."; return true; @@ -684,7 +728,7 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) */ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) { - /* Avoid loading the authoritative marker for syncs. */ + // Avoid loading the authoritative marker for syncs at all cost. if (Utility::BaseName(file) == ".authoritative") return; @@ -716,7 +760,7 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& /* Calculate a checksum for each file (and a global one later). * - * IMPORTANT: Ignore the .authoritative file, this will not be synced. + * IMPORTANT: Ignore the .authoritative file above, this must not be synced. * */ config.Checksums->Set(relativePath, GetChecksum(content)); } @@ -738,5 +782,4 @@ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& confi config.UpdateV2->CopyTo(result); return result; -} - +} \ No newline at end of file From db4cc137706b38fcecb51771bfff490f4ad0c1ca Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 19 Jun 2019 16:09:16 +0200 Subject: [PATCH 35/37] Config Sync: Only log config files for stage, no metadata --- lib/remote/apilistener-filesync.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index f1ed77576..42c4737fe 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -447,8 +447,10 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D String path = stageConfigZoneDir + "/" + kv.first; - Log(LogInformation, "ApiListener") - << "Stage: Updating received configuration file '" << path << "' for zone '" << zoneName << "'."; + if (Utility::Match("*.conf", path)) { + Log(LogInformation, "ApiListener") + << "Stage: Updating received configuration file '" << path << "' for zone '" << zoneName << "'."; + } // Sync string content only. String content = kv.second; @@ -782,4 +784,4 @@ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& confi config.UpdateV2->CopyTo(result); return result; -} \ No newline at end of file +} From 08a47600be3e9be87b702aa81157a9eb013b27ab Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 19 Jun 2019 17:00:50 +0200 Subject: [PATCH 36/37] Config sync: Only copy paths to prod which are actually there Stored files may be removed by external sources. --- lib/remote/apilistener-filesync.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 42c4737fe..f3d5a5d48 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -547,6 +547,9 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, // Copy all synced configuration files from stage to production. for (const String& path : relativePaths) { + if (!Utility::PathExists(path)) + continue; + Log(LogInformation, "ApiListener") << "Copying file '" << path << "' from config sync staging to production zones directory."; From 1c9a540fc8745879846c6266c41f950cf13950ad Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 19 Jun 2019 17:04:51 +0200 Subject: [PATCH 37/37] Docs: Add cluster config sync to technical concepts & upgrading --- doc/16-upgrading-icinga-2.md | 30 +++++- doc/19-technical-concepts.md | 182 +++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) diff --git a/doc/16-upgrading-icinga-2.md b/doc/16-upgrading-icinga-2.md index 39d35a5ab..67d79f4e2 100644 --- a/doc/16-upgrading-icinga-2.md +++ b/doc/16-upgrading-icinga-2.md @@ -96,7 +96,33 @@ feature with the `cipher_list` attribute. In case that one of these ciphers is marked as insecure in the future, please let us know with an issue on GitHub. -### HA-aware Features +### Cluster + +#### Config Sync + +2.11 overhauls the cluster config sync in many ways. This includes the +following under the hood: + +- Synced configuration files are not immediately put into production, but left inside a stage. +- Unsuccessful config validation never puts the config into production, additional logging and API states are available. +- Zone directories which are not configured in zones.conf, are not included anymore on secondary master/satellites/clients. +- Synced config change calculation use checksums instead of timestamps to trigger validation/reload. This is more safe, and the usage of timestamps is now deprecated. +- Don't allow parallel cluster syncs to avoid race conditions with overridden files. +- Deleted directories and files are now purged, previous versions had a bug. + +Whenever a newer child endpoint receives a configuration update without +checksums, it will log a warning. + +``` +Received configuration update without checksums from parent endpoint satellite1. This behaviour is deprecated. Please upgrade the parent endpoint to 2.11+ +``` + +This is a gentle reminder to upgrade the master and satellites first, +prior to installing new clients/agents. + +Technical details are available in the [technical concepts](19-technical-concepts.md#technical-concepts-cluster-config-sync) chapter. + +#### HA-aware Features v2.11 introduces additional HA functionality similar to the DB IDO feature. This enables the feature being active only on one endpoint while the other @@ -182,7 +208,7 @@ constant in [constants.conf](04-configuring-icinga-2.md#constants-conf) instead. ### REST API -#### Actions +#### Actions The [schedule-downtime](12-icinga2-api.md#icinga2-api-actions-schedule-downtime-host-all-services) action supports the `all_services` parameter for Host types. Defaults to false. diff --git a/doc/19-technical-concepts.md b/doc/19-technical-concepts.md index e6483132a..12ed44f4b 100644 --- a/doc/19-technical-concepts.md +++ b/doc/19-technical-concepts.md @@ -813,6 +813,188 @@ Icinga 2 v2.9+ adds more performance metrics for these values: * `sum_bytes_sent_per_second` and `sum_bytes_received_per_second` +### Config Sync + +The visible feature for the user is to put configuration files in `/etc/icinga2/zones.d/` +and have them synced automatically to all involved zones and endpoints. + +This not only includes host and service objects being checked +in a satellite zone, but also additional config objects such as +commands, groups, timeperiods and also templates. + +Additional thoughts and complexity added: + +- Putting files into zone directory names removes the burden to set the `zone` attribute on each object in this directory. This is done automatically by the config compiler. +- Inclusion of `zones.d` happens automatically, the user shouldn't be bothered about this. +- Before the REST API was created, only static configuration files in `/etc/icinga2/zones.d` existed. With the addition of config packages, additional `zones.d` targets must be registered (e.g. used by the Director) +- Only one config master is allowed. This one identifies itself with configuration files in `/etc/icinga2/zones.d`. This is not necessarily the zone master seen in the debug logs, that one is important for message routing internally. +- Objects and templates which cannot be bound into a specific zone (e.g. hosts in the satellite zone) must be made available "globally". +- Users must be able to deny the synchronisation of specific zones, e.g. for security reasons. + +#### Config Sync: Config Master + +All zones must be configured and included in the `zones.conf` config file beforehand. +The zone names are the identifier for the directories underneath the `/etc/icinga2/zones.d` +directory. If a zone is not configured, it will not be included in the config sync - keep this +in mind for troubleshooting. + +When the config master starts, the content of `/etc/icinga2/zones.d` is automatically +included. There's no need for an additional entry in `icinga2.conf` like `conf.d`. +You can verify this by running the config validation on debug level: + +``` +icinga2 daemon -C -x debug | grep 'zones.d' + +[2019-06-19 15:16:19 +0200] notice/ConfigCompiler: Compiling config file: /etc/icinga2/zones.d/global-templates/commands.conf +``` + +Once the config validation succeeds, the startup routine for the daemon +copies the files into the "production" directory in `/var/lib/icinga2/api/zones`. +This directory is used for all endpoints where Icinga stores the received configuration. +With the exception of the config master retrieving this from `/etc/icinga2/zones.d` instead. + +These operations are logged for better visibility. + +``` +[2019-06-19 15:26:38 +0200] information/ApiListener: Copying 1 zone configuration files for zone 'global-templates' to '/var/lib/icinga2/api/zones/global-templates'. +[2019-06-19 15:26:38 +0200] information/ApiListener: Updating configuration file: /var/lib/icinga2/api/zones/global-templates//_etc/commands.conf +``` + +The master is finished at this point. Depending on the cluster configuration, +the next iteration is a connected endpoint after successful TLS handshake and certificate +authentication. + +It calls `SendConfigUpdate(client)` which sends the [config::Update](19-technical-concepts.md#technical-concepts-json-rpc-messages-config-update) +JSON-RPC message including all required zones and their configuration file content. + + +#### Config Sync: Receive Config + +The secondary master endpoint and endpoints in a child zone will be connected to the config +master. The endpoint receives the [config::Update](19-technical-concepts.md#technical-concepts-json-rpc-messages-config-update) +JSON-RPC message and processes the content in `ConfigUpdateHandler()`. This method checks +whether config should be accepted. In addition to that, it locks a local mutex to avoid race conditions +with multiple syncs in parallel. + +After that, the received configuration content is analysed. + +> **Note** +> +> The cluster design allows that satellite endpoints may connect to the secondary master first. +> There is no immediate need to always connect to the config master first, especially since +> the satellite endpoints don't know that. +> +> The secondary master not only stores the master zone config files, but also all child zones. +> This is also the case for any HA enabled zone with more than one endpoint. + + +2.11 puts the received configuration files into a staging directory in +`/var/lib/icinga2/api/zones-stage`. Previous versions directly wrote the +files into production which could have led to broken configuration on the +next manual restart. + +``` +[2019-06-19 16:08:29 +0200] information/ApiListener: New client connection for identity 'master1' to [127.0.0.1]:5665 +[2019-06-19 16:08:30 +0200] information/ApiListener: Applying config update from endpoint 'master1' of zone 'master'. +[2019-06-19 16:08:30 +0200] information/ApiListener: Received configuration for zone 'agent' from endpoint 'master1'. Comparing the checksums. +[2019-06-19 16:08:30 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/agent//_etc/host.conf' for zone 'agent'. +[2019-06-19 16:08:30 +0200] information/ApiListener: Applying configuration file update for path '/var/lib/icinga2/api/zones-stage/agent' (176 Bytes). +[2019-06-19 16:08:30 +0200] information/ApiListener: Received configuration for zone 'master' from endpoint 'master1'. Comparing the checksums. +[2019-06-19 16:08:30 +0200] information/ApiListener: Applying configuration file update for path '/var/lib/icinga2/api/zones-stage/master' (17 Bytes). +[2019-06-19 16:08:30 +0200] information/ApiListener: Received configuration from endpoint 'master1' is different to production, triggering validation and reload. +``` + +It then validates the received configuration in its own config stage. There is +an parameter override in place which disables the automatic inclusion of the production +config in `/var/lib/icinga2/api/zones`. + +Once completed, the reload is triggered. This follows the same configurable timeout +as with the global reload. + +``` +[2019-06-19 16:52:26 +0200] information/ApiListener: Config validation for stage '/var/lib/icinga2/api/zones-stage/' was OK, replacing into '/var/lib/icinga2/api/zones/' and triggering reload. +[2019-06-19 16:52:27 +0200] information/Application: Got reload command: Started new instance with PID '19945' (timeout is 300s). +[2019-06-19 16:52:28 +0200] information/Application: Reload requested, letting new process take over. +``` + +Whenever the staged configuration validation fails, Icinga logs this including a reference +to the startup log file which includes additional errors. + +``` +[2019-06-19 15:45:27 +0200] critical/ApiListener: Config validation failed for staged cluster config sync in '/var/lib/icinga2/api/zones-stage/'. Aborting. Logs: '/var/lib/icinga2/api/zones-stage//startup.log' +``` + + +#### Config Sync: Changes and Reload + +Whenever a new configuration is received, it is validated and upon success, the +daemon automatically reloads. While the daemon continues with checks, the reload +cannot hand over open TCP connections. That being said, reloading the daemon everytime +a configuration is synchronized would lead into many not connected endpoints. + +Therefore the cluster config sync checks whether the configuration files actually +changed, and will only trigger a reload when such a change happened. + +2.11 calculates a checksum from each file content and compares this to the +production configuration. Previous versions used additional metadata with timestamps from +files which sometimes led to problems with asynchronous dates. + +> **Note** +> +> For compatibility reasons, the timestamp metadata algorithm is still intact, e.g. +> when the client is 2.11 already, but the parent endpoint is still on 2.10. + +Icinga logs a warning when this happens. + +``` +Received configuration update without checksums from parent endpoint satellite1. This behaviour is deprecated. Please upgrade the parent endpoint to 2.11+ +``` + + +The debug log provides more details on the actual checksums and checks. Future output +may change, use this solely for troubleshooting and debugging whenever the cluster +config sync fails. + +``` +[2019-06-19 16:13:16 +0200] information/ApiListener: Received configuration for zone 'agent' from endpoint 'master1'. Comparing the checksums. +[2019-06-19 16:13:16 +0200] debug/ApiListener: Checking for config change between stage and production. Old (3): '{"/.checksums":"7ede1276a9a32019c1412a52779804a976e163943e268ec4066e6b6ec4d15d73","/.timestamp":"ec4354b0eca455f7c2ca386fddf5b9ea810d826d402b3b6ac56ba63b55c2892c","/_etc/host.conf":"35d4823684d83a5ab0ca853c9a3aa8e592adfca66210762cdf2e54339ccf0a44"}' vs. new (3): '{"/.checksums":"84a586435d732327e2152e7c9b6d85a340cc917b89ae30972042f3dc344ea7cf","/.timestamp":"0fd6facf35e49ab1b2a161872fa7ad794564eba08624373d99d31c32a7a4c7d3","/_etc/host.conf":"0d62075e89be14088de1979644b40f33a8f185fcb4bb6ff1f7da2f63c7723fcb"}'. +[2019-06-19 16:13:16 +0200] debug/ApiListener: Checking /_etc/host.conf for checksum: 35d4823684d83a5ab0ca853c9a3aa8e592adfca66210762cdf2e54339ccf0a44 +[2019-06-19 16:13:16 +0200] debug/ApiListener: Path '/_etc/host.conf' doesn't match old checksum '0d62075e89be14088de1979644b40f33a8f185fcb4bb6ff1f7da2f63c7723fcb' with new checksum '35d4823684d83a5ab0ca853c9a3aa8e592adfca66210762cdf2e54339ccf0a44'. +``` + + +#### Config Sync: Trust + +The config sync follows the "top down" approach, where the master endpoint in the master +zone is allowed to synchronize configuration to the child zone, e.g. the satellite zone. + +Endpoints in the same zone, e.g. a secondary master, receive configuration for the same +zone and all child zones. + +Endpoints in the satellite zone trust the parent zone, and will accept the pushed +configuration via JSON-RPC cluster messages. By default, this is disabled and must +be enabled with the `accept_config` attribute in the ApiListener feature (manually or with CLI +helpers). + +The satellite zone will not only accept zone configuration for its own zone, but also +all configured child zones. That is why it is important to configure the zone hierarchy +on the satellite as well. + +Child zones are not allowed to sync configuration up to the parent zone. Each Icinga instance +evaluates this in startup and knows on endpoint connect which config zones need to be synced. + + +Global zones have a special trust relationship: They are synced to all child zones, be it +a satellite zone or client zone. Since checkable objects such as a Host or a Service object +must have only one endpoint as authority, they cannot be put into a global zone (denied by +the config compiler). + +Apply rules and templates are allowed, since they are evaluated in the endpoint which received +the synced configuration. Keep in mind that there may be differences on the master and the satellite +when e.g. hostgroup membership is used for assign where expressions, but the groups are only +available on the master. + + ## TLS Network IO ### TLS Connection Handling