From e1bc4d474fa0ea3221c362b63e9c2ae2467c7dd9 Mon Sep 17 00:00:00 2001
From: "Alexander A. Klimov" <alexander.klimov@icinga.com>
Date: Thu, 14 Jan 2021 12:00:11 +0100
Subject: [PATCH] On check_timeout first send SIGTERM

... to allow check plugins to terminate gracefully.

refs #6162
---
 lib/base/process.cpp | 52 ++++++++++++++++++++++++++++++++++++++------
 lib/base/process.hpp |  5 +++++
 2 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/lib/base/process.cpp b/lib/base/process.cpp
index c16eaa7c1..ee9a324d4 100644
--- a/lib/base/process.cpp
+++ b/lib/base/process.cpp
@@ -50,10 +50,13 @@ static boost::once_flag l_SpawnHelperOnceFlag = BOOST_ONCE_INIT;
 
 Process::Process(Process::Arguments arguments, Dictionary::Ptr extraEnvironment)
 	: m_Arguments(std::move(arguments)), m_ExtraEnvironment(std::move(extraEnvironment)),
-	  m_Timeout(600), m_AdjustPriority(false), m_ResultAvailable(false)
+	  m_Timeout(600)
 #ifdef _WIN32
 	, m_ReadPending(false), m_ReadFailed(false), m_Overlapped()
+#else /* _WIN32 */
+	, m_SentSigterm(false)
 #endif /* _WIN32 */
+	, m_AdjustPriority(false), m_ResultAvailable(false)
 {
 #ifdef _WIN32
 	m_Overlapped.hEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);
@@ -658,7 +661,7 @@ void Process::IOThreadProc(int tid)
 #endif /* _WIN32 */
 
 				if (process->m_Timeout != 0) {
-					double delta = process->m_Timeout - (now - process->m_Result.ExecutionStart);
+					double delta = process->GetNextTimeout() - (now - process->m_Result.ExecutionStart);
 
 					if (timeout == -1 || delta < timeout)
 						timeout = delta;
@@ -716,7 +719,7 @@ void Process::IOThreadProc(int tid)
 				bool is_timeout = false;
 
 				if (it->second->m_Timeout != 0) {
-					double timeout = it->second->m_Result.ExecutionStart + it->second->m_Timeout;
+					double timeout = it->second->m_Result.ExecutionStart + it->second->GetNextTimeout();
 
 					if (timeout < now)
 						is_timeout = true;
@@ -1029,15 +1032,42 @@ bool Process::DoEvents()
 #endif /* _WIN32 */
 
 	if (m_Timeout != 0) {
-		double timeout = m_Result.ExecutionStart + m_Timeout;
+		auto now (Utility::GetTime());
 
-		if (timeout < Utility::GetTime()) {
+#ifndef _WIN32
+		{
+			auto timeout (GetNextTimeout());
+			auto deadline (m_Result.ExecutionStart + timeout);
+
+			if (deadline < now && !m_SentSigterm) {
+				Log(LogWarning, "Process")
+					<< "Terminating process " << m_PID << " (" << PrettyPrintArguments(m_Arguments)
+					<< ") after timeout of " << timeout << " seconds";
+
+				m_OutputStream << "<Timeout exceeded.>";
+
+				int error = ProcessKill(m_Process, SIGTERM);
+				if (error) {
+					Log(LogWarning, "Process")
+						<< "Couldn't terminate the process " << m_PID << " (" << PrettyPrintArguments(m_Arguments)
+						<< "): [errno " << error << "] " << strerror(error);
+				}
+
+				m_SentSigterm = true;
+			}
+		}
+#endif /* _WIN32 */
+
+		auto timeout (GetNextTimeout());
+		auto deadline (m_Result.ExecutionStart + timeout);
+
+		if (deadline < now) {
 			Log(LogWarning, "Process")
 				<< "Killing process group " << m_PID << " (" << PrettyPrintArguments(m_Arguments)
-				<< ") after timeout of " << m_Timeout << " seconds";
+				<< ") after timeout of " << timeout << " seconds";
 
-			m_OutputStream << "<Timeout exceeded.>";
 #ifdef _WIN32
+			m_OutputStream << "<Timeout exceeded.>";
 			TerminateProcess(m_Process, 3);
 #else /* _WIN32 */
 			int error = ProcessKill(-m_Process, SIGKILL);
@@ -1155,3 +1185,11 @@ int Process::GetTID() const
 	return (reinterpret_cast<uintptr_t>(this) / sizeof(void *)) % IOTHREADS;
 }
 
+double Process::GetNextTimeout() const
+{
+#ifdef _WIN32
+	return m_Timeout;
+#else /* _WIN32 */
+	return m_SentSigterm ? m_Timeout * 1.1 : m_Timeout;
+#endif /* _WIN32 */
+}
diff --git a/lib/base/process.hpp b/lib/base/process.hpp
index 53ee2bb5b..d83ba6ee7 100644
--- a/lib/base/process.hpp
+++ b/lib/base/process.hpp
@@ -82,6 +82,10 @@ private:
 	Dictionary::Ptr m_ExtraEnvironment;
 
 	double m_Timeout;
+#ifndef _WIN32
+	bool m_SentSigterm;
+#endif /* _WIN32 */
+
 	bool m_AdjustPriority;
 
 	ProcessHandle m_Process;
@@ -105,6 +109,7 @@ private:
 	static void IOThreadProc(int tid);
 	bool DoEvents();
 	int GetTID() const;
+	double GetNextTimeout() const;
 };
 
 }