Add API & Cluster stats to /v1/status & icinga check performance metrics

refs #5133

lib/db_ido_mysql/idomysqlconnection.cpp

@@ -53,13 +53,15 @@ void IdoMysqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
 	Dictionary::Ptr nodes = new Dictionary();
 
 	for (const IdoMysqlConnection::Ptr& idomysqlconnection : ConfigType::GetObjectsByType<IdoMysqlConnection>()) {
-		size_t items = idomysqlconnection->m_QueryQueue.GetLength();
+		size_t queryQueueItems = idomysqlconnection->m_QueryQueue.GetLength();
+		double queryQueueItemRate = idomysqlconnection->m_QueryQueue.GetTaskCount(60) / 60.0;
 
 		Dictionary::Ptr stats = new Dictionary();
 		stats->Set("version", idomysqlconnection->GetSchemaVersion());
 		stats->Set("instance_name", idomysqlconnection->GetInstanceName());
 		stats->Set("connected", idomysqlconnection->GetConnected());
-		stats->Set("query_queue_items", items);
+		stats->Set("query_queue_items", queryQueueItems);
+		stats->Set("query_queue_item_rate", queryQueueItemRate);
 
 		nodes->Set(idomysqlconnection->GetName(), stats);
 
@@ -67,7 +69,8 @@ void IdoMysqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
 		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_1min", idomysqlconnection->GetQueryCount(60)));
 		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_5mins", idomysqlconnection->GetQueryCount(5 * 60)));
 		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_15mins", idomysqlconnection->GetQueryCount(15 * 60)));
-		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_items", items));
+		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_items", queryQueueItems));
+		perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_item_rate", queryQueueItemRate));
 	}
 
 	status->Set("idomysqlconnection", nodes);

lib/db_ido_pgsql/idopgsqlconnection.cpp

@@ -57,13 +57,15 @@ void IdoPgsqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
 	Dictionary::Ptr nodes = new Dictionary();
 
 	for (const IdoPgsqlConnection::Ptr& idopgsqlconnection : ConfigType::GetObjectsByType<IdoPgsqlConnection>()) {
-		size_t items = idopgsqlconnection->m_QueryQueue.GetLength();
+		size_t queryQueueItems = idopgsqlconnection->m_QueryQueue.GetLength();
+		double queryQueueItemRate = idopgsqlconnection->m_QueryQueue.GetTaskCount(60) / 60.0;
 
 		Dictionary::Ptr stats = new Dictionary();
 		stats->Set("version", idopgsqlconnection->GetSchemaVersion());
-		stats->Set("connected", idopgsqlconnection->GetConnected());
 		stats->Set("instance_name", idopgsqlconnection->GetInstanceName());
-		stats->Set("query_queue_items", items);
+		stats->Set("connected", idopgsqlconnection->GetConnected());
+		stats->Set("query_queue_items", queryQueueItems);
+		stats->Set("query_queue_item_rate", queryQueueItemRate);
 
 		nodes->Set(idopgsqlconnection->GetName(), stats);
 
@@ -71,7 +73,8 @@ void IdoPgsqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
 		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_1min", idopgsqlconnection->GetQueryCount(60)));
 		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_5mins", idopgsqlconnection->GetQueryCount(5 * 60)));
 		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_15mins", idopgsqlconnection->GetQueryCount(15 * 60)));
-		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_items", items));
+		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_items", queryQueueItems));
+		perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_item_rate", queryQueueItemRate));
 	}
 
 	status->Set("idopgsqlconnection", nodes);

lib/perfdata/influxdbwriter.cpp

@@ -70,11 +70,13 @@ void InfluxdbWriter::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr&)
 
 	for (const InfluxdbWriter::Ptr& influxdbwriter : ConfigType::GetObjectsByType<InfluxdbWriter>()) {
 		size_t workQueueItems = influxdbwriter->m_WorkQueue.GetLength();
+		double workQueueItemRate = influxdbwriter->m_WorkQueue.GetTaskCount(60) / 60.0;
 		size_t dataBufferItems = influxdbwriter->m_DataBuffer.size();
 
 		//TODO: Collect more stats
 		Dictionary::Ptr stats = new Dictionary();
 		stats->Set("work_queue_items", workQueueItems);
+		stats->Set("work_queue_item_rate", workQueueItemRate);
 		stats->Set("data_buffer_items", dataBufferItems);
 
 		nodes->Set(influxdbwriter->GetName(), stats);

lib/remote/apilistener.cpp

@@ -30,6 +30,7 @@
 #include "base/logger.hpp"
 #include "base/objectlock.hpp"
 #include "base/stdiostream.hpp"
+#include "base/perfdatavalue.hpp"
 #include "base/application.hpp"
 #include "base/context.hpp"
 #include "base/statsfunction.hpp"
@@ -292,7 +293,7 @@ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint)
 	String host = endpoint->GetHost();
 	String port = endpoint->GetPort();
 
-	Log(LogInformation, "JsonRpcConnection")
+	Log(LogInformation, "ApiListener")
 	    << "Reconnecting to API endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'";
 
 	TcpSocket::Ptr client = new TcpSocket();
@@ -1068,7 +1069,7 @@ void ApiListener::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& per
 
 	ObjectLock olock(stats.second);
 	for (const Dictionary::Pair& kv : stats.second)
-		perfdata->Add("'api_" + kv.first + "'=" + Convert::ToString(kv.second));
+		perfdata->Add(new PerfdataValue("api_" + kv.first, kv.second));
 
 	status->Set("api", stats.first);
 }
@@ -1152,10 +1153,50 @@ std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus(void)
 
 	status->Set("zones", connectedZones);
 
+	/* connection stats */
+	size_t jsonRpcClients = GetAnonymousClients().size();
+	size_t httpClients = GetHttpClients().size();
+	size_t workQueueItems = JsonRpcConnection::GetWorkQueueLength();
+	size_t workQueueCount = JsonRpcConnection::GetWorkQueueCount();
+	size_t syncQueueItems = m_SyncQueue.GetLength();
+	size_t relayQueueItems = m_RelayQueue.GetLength();
+	double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate();
+	double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0;
+	double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0;
+
+	Dictionary::Ptr jsonRpc = new Dictionary();
+	jsonRpc->Set("clients", jsonRpcClients);
+	jsonRpc->Set("work_queue_items", workQueueItems);
+	jsonRpc->Set("work_queue_count", workQueueCount);
+	jsonRpc->Set("sync_queue_items", syncQueueItems);
+	jsonRpc->Set("relay_queue_items", relayQueueItems);
+
+	jsonRpc->Set("work_queue_item_rate", workQueueItemRate);
+	jsonRpc->Set("sync_queue_item_rate", syncQueueItemRate);
+	jsonRpc->Set("relay_queue_item_rate", relayQueueItemRate);
+
+	Dictionary::Ptr http = new Dictionary();
+	http->Set("clients", httpClients);
+
+	status->Set("json_rpc", jsonRpc);
+	status->Set("http", http);
+
+	/* performance data */
 	perfdata->Set("num_endpoints", allEndpoints);
 	perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
 	perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
 
+	perfdata->Set("num_json_rpc_clients", jsonRpcClients);
+	perfdata->Set("num_http_clients", httpClients);
+	perfdata->Set("num_json_rpc_work_queue_items", workQueueItems);
+	perfdata->Set("num_json_rpc_work_queue_count", workQueueCount);
+	perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems);
+	perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems);
+
+	perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate);
+	perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate);
+	perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate);
+
 	return std::make_pair(status, perfdata);
 }
 

lib/remote/jsonrpcconnection.cpp

@@ -340,3 +340,30 @@ void JsonRpcConnection::TimeoutTimerHandler(void)
 	}
 }
 
+int JsonRpcConnection::GetWorkQueueCount(void)
+{
+	return l_JsonRpcConnectionWorkQueueCount;
+}
+
+int JsonRpcConnection::GetWorkQueueLength(void)
+{
+	size_t itemCount = 0;
+
+	for (size_t i = 0; i < l_JsonRpcConnectionWorkQueueCount; i++) {
+		itemCount += l_JsonRpcConnectionWorkQueues[i].GetLength();
+	}
+
+	return itemCount;
+}
+
+double JsonRpcConnection::GetWorkQueueRate(void)
+{
+	double rate = 0.0;
+
+	for (size_t i = 0; i < l_JsonRpcConnectionWorkQueueCount; i++) {
+		rate += l_JsonRpcConnectionWorkQueues[i].GetTaskCount(60) / 60.0;
+	}
+
+	return rate / l_JsonRpcConnectionWorkQueueCount;
+}
+

lib/remote/jsonrpcconnection.hpp

@@ -71,6 +71,10 @@ public:
 	static void HeartbeatTimerHandler(void);
 	static Value HeartbeatAPIHandler(const intrusive_ptr<MessageOrigin>& origin, const Dictionary::Ptr& params);
 
+	static int GetWorkQueueCount(void);
+	static int GetWorkQueueLength(void);
+	static double GetWorkQueueRate(void);
+
 private:
 	int m_ID;
 	String m_Identity;