Merge branch 'feature/threadpool-bug-5050' into next

Fixes #5050

components/checker/checkercomponent.cpp

@@ -140,7 +140,7 @@ void CheckerComponent::CheckThreadProc(void)
 		Log(LogDebug, "checker", "Executing service check for '" + service->GetName() + "'");
 
 		CheckerComponent::Ptr self = GetSelf();
-		Utility::QueueAsyncCallback(boost::bind(&CheckerComponent::ExecuteCheckHelper, self, service));
+		m_Pool.Post(boost::bind(&CheckerComponent::ExecuteCheckHelper, self, service));
 
 		lock.lock();
 	}

components/checker/checkercomponent.h

@@ -92,6 +92,8 @@ private:
 	void NextCheckChangedHandler(const Service::Ptr& service);
 
 	void RescheduleCheckTimer(void);
+
+	ThreadPool m_Pool;
 };
 
 }

etc/init.d/icinga2.cmake

@@ -54,7 +54,9 @@ start() {
         fi
 
         echo "Starting Icinga 2: "
-        ulimit -n 8192
+        ulimit -n 32768
+	ulimit -s 512
+	ulimit -u 16384
         $DAEMON -c $ICINGA2_CONFIG_FILE -d -e $ICINGA2_ERROR_LOG -u $ICINGA2_USER -g $ICINGA2_GROUP
 
         echo "Done"

lib/base/threadpool.cpp

@@ -23,6 +23,7 @@
 #include "base/debug.h"
 #include "base/utility.h"
 #include "base/scriptvariable.h"
+#include "base/application.h"
 #include <sstream>
 #include <iostream>
 #include <boost/bind.hpp>
@@ -252,9 +253,13 @@ void ThreadPool::ManagerThreadProc(void)
 				if (alive + tthreads < 8)
 					tthreads = 8 - alive;
 
+				/* Don't kill more than 8 threads at once. */
+				if (tthreads < -8)
+					tthreads = -8;
+
 				/* Spawn more workers if there are outstanding work items. */
 				if (tthreads > 0 && pending > 0)
-					tthreads = 8;
+					tthreads = (Utility::GetTime() - Application::GetStartTime() < 300) ? 128 : 8;
 
 				std::ostringstream msgbuf;
 				msgbuf << "Thread pool; current: " << alive << "; adjustment: " << tthreads;
@@ -276,7 +281,7 @@ void ThreadPool::ManagerThreadProc(void)
 		}
 
 		std::ostringstream msgbuf;
-		msgbuf << "Pending tasks: " << pending << "; Average latency: "
+		msgbuf << "Pool #" << m_ID << ": Pending tasks: " << pending << "; Average latency: "
 		    << (long)(avg_latency * 1000) << "ms"
 		    << "; Max latency: " << (long)(max_latency * 1000) << "ms"
 		    << "; Threads: " << alive

lib/base/threadpool.h

@@ -73,7 +73,7 @@ private:
 	static int m_NextID;
 
 	boost::thread_group m_Threads;
-	ThreadStats m_ThreadStats[512];
+	ThreadStats m_ThreadStats[4096];
 
 	double m_WaitTime;
 	double m_ServiceTime;

lib/icinga/service-check.cpp

@@ -108,7 +108,7 @@ void Service::UpdateNextCheck(void)
 
 	double interval;
 
-	if (GetStateType() == StateTypeSoft)
+	if (GetStateType() == StateTypeSoft && GetLastCheckResult() != NULL)
 		interval = GetRetryInterval();
 	else
 		interval = GetCheckInterval();
@@ -459,6 +459,7 @@ void Service::ExecuteCheck(void)
 	}
 
 	/* keep track of scheduling info in case the check type doesn't provide its own information */
+	double scheduled_start = GetNextCheck();
 	double before_check = Utility::GetTime();
 
 	Service::Ptr self = GetSelf();
@@ -490,16 +491,16 @@ void Service::ExecuteCheck(void)
 	double after_check = Utility::GetTime();
 
 	if (result) {
-		if (!result->GetScheduleStart() == 0)
-			result->SetScheduleStart(before_check);
+		if (result->GetScheduleStart() == 0)
+			result->SetScheduleStart(scheduled_start);
 
-		if (!result->GetScheduleEnd() == 0)
+		if (result->GetScheduleEnd() == 0)
 			result->SetScheduleEnd(after_check);
 
-		if (!result->GetExecutionStart() == 0)
+		if (result->GetExecutionStart() == 0)
 			result->SetExecutionStart(before_check);
 
-		if (!result->GetExecutionEnd() == 0)
+		if (result->GetExecutionEnd() == 0)
 			result->SetExecutionEnd(after_check);
 	}