Fix incorrect re-scheduling behavior for command_endpoint checks

refs #8137

lib/checker/checkercomponent.cpp

@@ -69,7 +69,6 @@ void CheckerComponent::OnConfigLoaded(void)
 	ConfigObject::OnActiveChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
 	ConfigObject::OnPausedChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
 
-	Checkable::OnNewCheckResult.connect(bind(&CheckerComponent::CheckResultHandler, this, _1));
 	Checkable::OnNextCheckChanged.connect(bind(&CheckerComponent::NextCheckChangedHandler, this, _1));
 }
 
@@ -122,7 +121,7 @@ void CheckerComponent::CheckThreadProc(void)
 
 		double wait = checkable->GetNextCheck() - Utility::GetTime();
 
-		if (wait > 0 || m_PendingCheckables.size() >= GetConcurrentChecks()) {
+		if (wait > 0 || Checkable::GetPendingChecks() >= GetConcurrentChecks()) {
 			/* Wait for the next check. */
 			m_CV.timed_wait(lock, boost::posix_time::milliseconds(wait * 1000));
 
@@ -216,6 +215,27 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
 
 		Log(LogCritical, "checker", output);
 	}
+
+	{
+		boost::mutex::scoped_lock lock(m_Mutex);
+
+		/* remove the object from the list of pending objects; if it's not in the
+		 * list this was a manual (i.e. forced) check and we must not re-add the
+		 * object to the list because it's already there. */
+		CheckerComponent::CheckableSet::iterator it;
+		it = m_PendingCheckables.find(checkable);
+		if (it != m_PendingCheckables.end()) {
+			m_PendingCheckables.erase(it);
+
+			if (checkable->IsActive())
+				m_IdleCheckables.insert(checkable);
+
+			m_CV.notify_all();
+		}
+	}
+
+	Log(LogDebug, "CheckerComponent")
+	    << "Check finished for object '" << checkable->GetName() << "'";
 }
 
 void CheckerComponent::ResultTimerHandler(void)
@@ -259,30 +279,6 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
 	}
 }
 
-void CheckerComponent::CheckResultHandler(const Checkable::Ptr& checkable)
-{
-	{
-		boost::mutex::scoped_lock lock(m_Mutex);
-
-		/* remove the object from the list of pending objects; if it's not in the
-		 * list this was a manual (i.e. forced) check and we must not re-add the
-		 * object to the list because it's already there. */
-		CheckerComponent::CheckableSet::iterator it;
-		it = m_PendingCheckables.find(checkable);
-		if (it != m_PendingCheckables.end()) {
-			m_PendingCheckables.erase(it);
-
-			if (checkable->IsActive())
-				m_IdleCheckables.insert(checkable);
-
-			m_CV.notify_all();
-		}
-	}
-
-	Log(LogDebug, "CheckerComponent")
-	    << "Check finished for object '" << checkable->GetName() << "'";
-}
-
 void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);

lib/checker/checkercomponent.hpp

@@ -97,7 +97,6 @@ private:
 	void AdjustCheckTimer(void);
 
 	void ObjectHandler(const ConfigObject::Ptr& object);
-	void CheckResultHandler(const Checkable::Ptr& checkable);
 	void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 
 	void RescheduleCheckTimer(void);

lib/icinga/checkable-check.cpp

@@ -41,6 +41,9 @@ boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, st
 boost::signals2::signal<void (const Checkable::Ptr&, NotificationType, const CheckResult::Ptr&, const String&, const String&)> Checkable::OnNotificationsRequested;
 boost::signals2::signal<void (const Checkable::Ptr&)> Checkable::OnNextCheckUpdated;
 
+boost::mutex Checkable::m_StatsMutex;
+int Checkable::m_PendingChecks = 0;
+
 CheckCommand::Ptr Checkable::GetCheckCommand(void) const
 {
 	return dynamic_pointer_cast<CheckCommand>(NavigateCheckCommandRaw());
@@ -515,3 +518,21 @@ void Checkable::UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type)
 		Log(LogWarning, "Checkable", "Unknown checkable type for statistic update.");
 	}
 }
+
+void Checkable::IncreasePendingChecks(void)
+{
+	boost::mutex::scoped_lock lock(m_StatsMutex);
+	m_PendingChecks++;
+}
+
+void Checkable::DecreasePendingChecks(void)
+{
+	boost::mutex::scoped_lock lock(m_StatsMutex);
+	m_PendingChecks--;
+}
+
+int Checkable::GetPendingChecks(void)
+{
+	boost::mutex::scoped_lock lock(m_StatsMutex);
+	return m_PendingChecks;
+}

lib/icinga/checkable.hpp

@@ -178,6 +178,10 @@ public:
 
 	virtual void ValidateCheckInterval(double value, const ValidationUtils& utils) override;
 
+	static void IncreasePendingChecks(void);
+	static void DecreasePendingChecks(void);
+	static int GetPendingChecks(void);
+
 protected:
 	virtual void Start(bool runtimeCreated) override;
 
@@ -186,6 +190,9 @@ private:
 	bool m_CheckRunning;
 	long m_SchedulingOffset;
 
+	static boost::mutex m_StatsMutex;
+	static int m_PendingChecks;
+
 	/* Downtimes */
 	std::set<Downtime::Ptr> m_Downtimes;
 	mutable boost::mutex m_DowntimeMutex;

lib/methods/pluginchecktask.cpp

@@ -52,6 +52,8 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	resolvers.push_back(std::make_pair("command", commandObj));
 	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
 
+	Checkable::IncreasePendingChecks();
+
 	PluginUtility::ExecuteCommand(commandObj, checkable, checkable->GetLastCheckResult(),
 	    resolvers, resolvedMacros, useResolvedMacros,
 	    boost::bind(&PluginCheckTask::ProcessFinishedHandler, checkable, cr, _1, _2));
@@ -59,6 +61,8 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 
 void PluginCheckTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Value& commandLine, const ProcessResult& pr)
 {
+	Checkable::DecreasePendingChecks();
+
 	if (pr.ExitStatus > 3) {
 		Process::Arguments parguments = Process::PrepareCommand(commandLine);
 		Log(LogWarning, "PluginCheckTask")