Implement ReloadTimeout constant and wait for enqueued checks on Stop()

doc/17-language-reference.md

@@ -440,6 +440,7 @@ Constant            | Description
 --------------------|-------------------
 Vars                |**Read-write.** Contains a dictionary with global custom attributes. Not set by default.
 NodeName            |**Read-write.** Contains the cluster node name. Set to the local hostname by default.
+ReloadTimeout       |**Read-write.** Defines the reload timeout for child processes. Defaults to `300s`.
 Environment         |**Read-write.** The name of the Icinga environment. Included in the SNI host name for outbound connections. Not set by default.
 RunAsUser           |**Read-write.** Defines the user the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig.
 RunAsGroup          |**Read-write.** Defines the group the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig.

lib/base/application.cpp

@@ -77,6 +77,14 @@ void Application::OnConfigLoaded()
 
 	ASSERT(m_Instance == nullptr);
 	m_Instance = this;
+
+	String reloadTimeout;
+
+	if (ScriptGlobal::Exists("ReloadTimeout"))
+		reloadTimeout = ScriptGlobal::Get("ReloadTimeout");
+
+	if (!reloadTimeout.IsEmpty())
+		Configuration::ReloadTimeout = Convert::ToDouble(reloadTimeout);
 }
 
 /**
@@ -401,8 +409,6 @@ static void ReloadProcessCallback(const ProcessResult& pr)
 
 pid_t Application::StartReloadProcess()
 {
-	Log(LogInformation, "Application", "Got reload command: Starting new instance.");
-
 	// prepare arguments
 	ArrayData args;
 	args.push_back(GetExePath(m_ArgV[0]));
@@ -422,9 +428,14 @@ pid_t Application::StartReloadProcess()
 #endif /* _WIN32 */
 
 	Process::Ptr process = new Process(Process::PrepareCommand(new Array(std::move(args))));
-	process->SetTimeout(300);
+	process->SetTimeout(Configuration::ReloadTimeout);
 	process->Run(&ReloadProcessCallback);
 
+	Log(LogInformation, "Application")
+		<< "Got reload command: Started new instance with PID '"
+		<< (unsigned long)(process->GetPID()) << "' (timeout is "
+		<< Configuration::ReloadTimeout << "s).";
+
 	return process->GetPID();
 }
 

lib/base/configuration.cpp

@@ -42,6 +42,7 @@ String Configuration::PidPath;
 String Configuration::PkgDataDir;
 String Configuration::PrefixDir;
 String Configuration::ProgramData;
+double Configuration::ReloadTimeout{300};
 int Configuration::RLimitFiles;
 int Configuration::RLimitProcesses;
 int Configuration::RLimitStack;
@@ -240,6 +241,16 @@ void Configuration::SetProgramData(const String& val, bool suppress_events, cons
 	HandleUserWrite("ProgramData", &Configuration::ProgramData, val, m_ReadOnly);
 }
 
+double Configuration::GetReloadTimeout() const
+{
+	return Configuration::ReloadTimeout;
+}
+
+void Configuration::SetReloadTimeout(double val, bool suppress_events, const Value& cookie)
+{
+	HandleUserWrite("ReloadTimeout", &Configuration::ReloadTimeout, val, m_ReadOnly);
+}
+
 int Configuration::GetRLimitFiles() const
 {
 	return Configuration::RLimitFiles;

lib/base/configuration.hpp

@@ -87,6 +87,9 @@ public:
 	String GetProgramData() const override;
 	void SetProgramData(const String& value, bool suppress_events = false, const Value& cookie = Empty) override;
 
+	double GetReloadTimeout() const override;
+	void SetReloadTimeout(double value, bool suppress_events = false, const Value& cookie = Empty) override;
+
 	int GetRLimitFiles() const override;
 	void SetRLimitFiles(int value, bool suppress_events = false, const Value& cookie = Empty) override;
 
@@ -147,6 +150,7 @@ public:
 	static String PkgDataDir;
 	static String PrefixDir;
 	static String ProgramData;
+	static double ReloadTimeout;
 	static int RLimitFiles;
 	static int RLimitProcesses;
 	static int RLimitStack;

lib/base/configuration.ti

@@ -111,6 +111,11 @@ abstract class Configuration
 		set;
 	};
 
+	[config, no_storage, virtual] double ReloadTimeout {
+		get;
+		set;
+	};
+
 	[config, no_storage, virtual] int RLimitFiles {
 		get;
 		set;

lib/checker/checkercomponent.cpp

@@ -84,18 +84,41 @@ void CheckerComponent::Start(bool runtimeCreated)
 
 void CheckerComponent::Stop(bool runtimeRemoved)
 {
-	Log(LogInformation, "CheckerComponent")
-		<< "'" << GetName() << "' stopped.";
-
 	{
 		boost::mutex::scoped_lock lock(m_Mutex);
 		m_Stopped = true;
 		m_CV.notify_all();
 	}
 
+	double wait = 0.0;
+
+	while (GetPendingCheckables() > 0) {
+		Log(LogDebug, "CheckerComponent")
+			<< "Waiting for running checks (" << GetPendingCheckables()
+			<< ") to finish. Waited for " << wait << " seconds now.";
+
+		Utility::Sleep(0.1);
+		wait += 0.1;
+
+		/* Pick a timeout slightly shorther than the process reload timeout. */
+		double waitMax = Configuration::ReloadTimeout - 30;
+		if (waitMax <= 0)
+			waitMax = 1;
+
+		if (wait > waitMax) {
+			Log(LogWarning, "CheckerComponent")
+				<< "Checks running too long for " << wait
+				<< " seconds, hard shutdown before reload timeout: " << Configuration::ReloadTimeout << ".";
+			break;
+		}
+	}
+
 	m_ResultTimer->Stop();
 	m_Thread.join();
 
+	Log(LogInformation, "CheckerComponent")
+		<< "'" << GetName() << "' stopped.";
+
 	ObjectImpl<CheckerComponent>::Stop(runtimeRemoved);
 }
 

lib/remote/configpackageutility.cpp

@@ -235,7 +235,7 @@ void ConfigPackageUtility::AsyncTryActivateStage(const String& packageName, cons
 	args->Add("ActiveStageOverride=" + packageName + ":" + stageName);
 
 	Process::Ptr process = new Process(Process::PrepareCommand(args));
-	process->SetTimeout(300);
+	process->SetTimeout(Configuration::ReloadTimeout);
 	process->Run(std::bind(&TryActivateStageCallback, _1, packageName, stageName, reload));
 }