Implement support for limiting the number of concurrent checks

fixes #8137

doc/6-object-types.md

@@ -189,7 +189,7 @@ Argument array `repeat_key = false`:
 
 ## <a id="objecttype-checkcomponent"></a> CheckerComponent
 
-The checker component is responsible for scheduling active checks. There are no configurable options.
+The checker component is responsible for scheduling active checks.
 
 Example:
 
@@ -197,6 +197,12 @@ Example:
 
     object CheckerComponent "checker" { }
 
+Configuration Attributes:
+
+  Name                |Description
+  --------------------|----------------
+  concurrent\_checks  |**Optional.** The maximum number of concurrent checks. Defaults to 512.
+
 ## <a id="objecttype-checkresultreader"></a> CheckResultReader
 
 Reads Icinga 1.x check results from a directory. This functionality is provided

lib/checker/checkercomponent.cpp

@@ -69,6 +69,7 @@ void CheckerComponent::OnConfigLoaded(void)
 	ConfigObject::OnActiveChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
 	ConfigObject::OnPausedChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
 
+	Checkable::OnNewCheckResult.connect(bind(&CheckerComponent::CheckResultHandler, this, _1));
 	Checkable::OnNextCheckChanged.connect(bind(&CheckerComponent::NextCheckChangedHandler, this, _1));
 }
 
@@ -121,7 +122,7 @@ void CheckerComponent::CheckThreadProc(void)
 
 		double wait = checkable->GetNextCheck() - Utility::GetTime();
 
-		if (wait > 0) {
+		if (wait > 0 || m_PendingCheckables.size() >= GetConcurrentChecks()) {
 			/* Wait for the next check. */
 			m_CV.timed_wait(lock, boost::posix_time::milliseconds(wait * 1000));
 
@@ -215,27 +216,6 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
 
 		Log(LogCritical, "checker", output);
 	}
-
-	{
-		boost::mutex::scoped_lock lock(m_Mutex);
-
-		/* remove the object from the list of pending objects; if it's not in the
-		 * list this was a manual (i.e. forced) check and we must not re-add the
-		 * object to the list because it's already there. */
-		CheckerComponent::CheckableSet::iterator it;
-		it = m_PendingCheckables.find(checkable);
-		if (it != m_PendingCheckables.end()) {
-			m_PendingCheckables.erase(it);
-
-			if (checkable->IsActive())
-				m_IdleCheckables.insert(checkable);
-
-			m_CV.notify_all();
-		}
-	}
-
-	Log(LogDebug, "CheckerComponent")
-	    << "Check finished for object '" << checkable->GetName() << "'";
 }
 
 void CheckerComponent::ResultTimerHandler(void)
@@ -279,6 +259,30 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
 	}
 }
 
+void CheckerComponent::CheckResultHandler(const Checkable::Ptr& checkable)
+{
+	{
+		boost::mutex::scoped_lock lock(m_Mutex);
+
+		/* remove the object from the list of pending objects; if it's not in the
+		 * list this was a manual (i.e. forced) check and we must not re-add the
+		 * object to the list because it's already there. */
+		CheckerComponent::CheckableSet::iterator it;
+		it = m_PendingCheckables.find(checkable);
+		if (it != m_PendingCheckables.end()) {
+			m_PendingCheckables.erase(it);
+
+			if (checkable->IsActive())
+				m_IdleCheckables.insert(checkable);
+
+			m_CV.notify_all();
+		}
+	}
+
+	Log(LogDebug, "CheckerComponent")
+	    << "Check finished for object '" << checkable->GetName() << "'";
+}
+
 void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);

lib/checker/checkercomponent.hpp

@@ -97,6 +97,7 @@ private:
 	void AdjustCheckTimer(void);
 
 	void ObjectHandler(const ConfigObject::Ptr& object);
+	void CheckResultHandler(const Checkable::Ptr& checkable);
 	void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 
 	void RescheduleCheckTimer(void);

lib/checker/checkercomponent.ti

@@ -26,6 +26,11 @@ namespace icinga
 
 class CheckerComponent : ConfigObject
 {
+	[config] int concurrent_checks {
+		default {{{
+			return 512;
+		}}}
+	};
 };
 
 }

lib/icinga/checkable-check.cpp

@@ -406,6 +406,10 @@ void Checkable::ExecuteCheck(void)
 {
 	CONTEXT("Executing check for object '" + GetName() + "'");
 
+	/* keep track of scheduling info in case the check type doesn't provide its own information */
+	double scheduled_start = GetNextCheck();
+	double before_check = Utility::GetTime();
+
 	UpdateNextCheck();
 
 	bool reachable = IsReachable();
@@ -424,10 +428,6 @@ void Checkable::ExecuteCheck(void)
 		SetLastReachable(reachable);
 	}
 
-	/* keep track of scheduling info in case the check type doesn't provide its own information */
-	double scheduled_start = GetNextCheck();
-	double before_check = Utility::GetTime();
-
 	CheckResult::Ptr cr = new CheckResult();
 
 	cr->SetScheduleStart(scheduled_start);