From a58a5feee3bea0cec5293cb672023dcc5b55f8fd Mon Sep 17 00:00:00 2001
From: "Alexander A. Klimov" <alexander.klimov@icinga.com>
Date: Wed, 14 Aug 2019 17:12:59 +0200
Subject: [PATCH 1/2] Introduce ThreadPool#GetPending()

---
 lib/base/threadpool.cpp |  2 +-
 lib/base/threadpool.hpp | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/lib/base/threadpool.cpp b/lib/base/threadpool.cpp
index d7819ef4a..26787ab52 100644
--- a/lib/base/threadpool.cpp
+++ b/lib/base/threadpool.cpp
@@ -6,7 +6,7 @@
 using namespace icinga;
 
 ThreadPool::ThreadPool(size_t threads)
-	: m_Threads(threads)
+	: m_Threads(threads), m_Pending(0)
 {
 	Start();
 }
diff --git a/lib/base/threadpool.hpp b/lib/base/threadpool.hpp
index deaf0439c..af351cd7a 100644
--- a/lib/base/threadpool.hpp
+++ b/lib/base/threadpool.hpp
@@ -3,6 +3,7 @@
 #ifndef THREADPOOL_H
 #define THREADPOOL_H
 
+#include "base/atomic.hpp"
 #include "base/exception.hpp"
 #include "base/logger.hpp"
 #include <cstddef>
@@ -14,6 +15,7 @@
 #include <boost/asio/thread_pool.hpp>
 #include <boost/thread/locks.hpp>
 #include <boost/thread/shared_mutex.hpp>
+#include <cstdint>
 
 namespace icinga
 {
@@ -52,7 +54,11 @@ public:
 		boost::shared_lock<decltype(m_Mutex)> lock (m_Mutex);
 
 		if (m_Pool) {
-			boost::asio::post(*m_Pool, [callback]() {
+			m_Pending.fetch_add(1);
+
+			boost::asio::post(*m_Pool, [this, callback]() {
+				m_Pending.fetch_sub(1);
+
 				try {
 					callback();
 				} catch (const std::exception& ex) {
@@ -70,10 +76,21 @@ public:
 		}
 	}
 
+	/**
+	 * Returns the amount of queued tasks not started yet.
+	 *
+	 * @returns amount of queued tasks.
+	 */
+	inline uint_fast64_t GetPending()
+	{
+		return m_Pending.load();
+	}
+
 private:
 	boost::shared_mutex m_Mutex;
 	std::unique_ptr<boost::asio::thread_pool> m_Pool;
 	size_t m_Threads;
+	Atomic<uint_fast64_t> m_Pending;
 };
 
 }

From 448a991404565878c0a6517a03c84e0624673402 Mon Sep 17 00:00:00 2001
From: "Alexander A. Klimov" <alexander.klimov@icinga.com>
Date: Wed, 14 Aug 2019 17:22:27 +0200
Subject: [PATCH 2/2] Expose metric current_pending_callbacks

---
 lib/icinga/cib.cpp              | 2 ++
 lib/methods/icingachecktask.cpp | 1 +
 2 files changed, 3 insertions(+)

diff --git a/lib/icinga/cib.cpp b/lib/icinga/cib.cpp
index b378bcd6f..555079ec9 100644
--- a/lib/icinga/cib.cpp
+++ b/lib/icinga/cib.cpp
@@ -4,6 +4,7 @@
 #include "icinga/host.hpp"
 #include "icinga/service.hpp"
 #include "icinga/clusterevents.hpp"
+#include "base/application.hpp"
 #include "base/objectlock.hpp"
 #include "base/utility.hpp"
 #include "base/perfdatavalue.hpp"
@@ -292,6 +293,7 @@ void CIB::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata) {
 	// Checker related stats
 	status->Set("remote_check_queue", ClusterEvents::GetCheckRequestQueueSize());
 	status->Set("current_concurrent_checks", Checkable::GetPendingChecks());
+	status->Set("current_pending_callbacks", Application::GetTP().GetPending());
 
 	CheckableCheckStatistics scs = CalculateServiceCheckStats();
 
diff --git a/lib/methods/icingachecktask.cpp b/lib/methods/icingachecktask.cpp
index e67d1ff94..4b7d0da01 100644
--- a/lib/methods/icingachecktask.cpp
+++ b/lib/methods/icingachecktask.cpp
@@ -75,6 +75,7 @@ void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 	perfdata->Add(new PerfdataValue("active_service_checks_15min", CIB::GetActiveServiceChecksStatistics(60 * 15)));
 	perfdata->Add(new PerfdataValue("passive_service_checks_15min", CIB::GetPassiveServiceChecksStatistics(60 * 15)));
 
+	perfdata->Add(new PerfdataValue("current_pending_callbacks", Application::GetTP().GetPending()));
 	perfdata->Add(new PerfdataValue("current_concurrent_checks", Checkable::GetPendingChecks()));
 	perfdata->Add(new PerfdataValue("remote_check_queue", ClusterEvents::GetCheckRequestQueueSize()));