Add OpenTSDB perfdata plugin

refs #7256

Signed-off-by: Gunnar Beutner <gunnar@beutner.name>

doc/3-monitoring-basics.md

@@ -2362,6 +2362,69 @@ Currently these events are processed:
 * State changes
 * Notifications
 
+### <a id="opentsdb-writer"></a> OpenTSDB Writer
+
+While there are some OpenTSDB collector scripts and daemons like tcollector available for
+Icinga 1.x it's more reasonable to directly process the check and plugin performance
+in memory in Icinga 2. Once there are new metrics available, Icinga 2 will directly
+write them to the defined TSDB TCP socket.
+
+You can enable the feature using
+
+    # icinga2 feature enable opentsdb
+
+By default the `OpenTsdbWriter` object expects the TSD to listen at
+`127.0.0.1` on port `4242`.
+
+The current naming schema is
+
+    icinga.host.<metricname>
+    icinga.service.<servicename>.<metricname>
+
+for host and service checks. The tag host is always applied.
+
+To make sure Icinga 2 writes a valid metric into OpenTSDB some characters are replaced
+with `_` in the target name:
+
+    \  (and space)
+
+The resulting name in OpenTSDB might look like:
+
+    www-01 / http-cert / response time
+    icinga.http_cert.response_time
+
+In addition to the performance data retrieved from the check plugin, Icinga 2 sends
+internal check statistic data to OpenTSDB:
+
+  metric             | description
+  -------------------|------------------------------------------
+  current_attempt    | current check attempt
+  max_check_attempts | maximum check attempts until the hard state is reached
+  reachable          | checked object is reachable
+  downtime_depth     | number of downtimes this object is in
+  execution_time     | check execution time
+  latency            | check latency
+  state              | current state of the checked object
+  state_type         | 0=SOFT, 1=HARD state
+
+While reachable, state and state_type are metrics for the host or service the
+other metrics follow the current naming schema
+
+    icinga.check.<metricname>
+
+with the following tags
+
+  tag     | description
+  --------|------------------------------------------
+  type    | the check type, one of [host, service]
+  host    | hostname, the check ran on
+  service | the service name (if type=service)
+
+> **Note**
+>
+> You might want to set the tsd.core.auto_create_metrics setting to `true`
+> in your opentsdb.conf configuration file.
+
 
 ## <a id="status-data"></a> Status Data
 

doc/5-object-types.md

@@ -744,6 +744,27 @@ Example with your custom [global constant](15-language-reference.md#constants) `
     host_name_template = GraphiteEnv + ".$host.name$"
     service_name_template = GraphiteEnv + ".$host.name$.$service.name$"
 
+## <a id="objecttype-opentsdbwriter"></a> OpenTsdbWriter
+
+Writes check result metrics and performance data to OpenTSDB.
+
+Example:
+
+    library "perfdata"
+
+    object OpenTsdbWriter "opentsdb" {
+      host = "127.0.0.1"
+      port = 4242
+    }
+
+Attributes:
+
+  Name            	|Description
+  ----------------------|----------------------
+  host            	|**Optional.** OpenTSDB host address. Defaults to '127.0.0.1'.
+  port            	|**Optional.** OpenTSDB port. Defaults to 4242.
+
+
 ## <a id="objecttype-gelfwriter"></a> GelfWriter
 
 Writes event log entries to a defined GELF receiver host (Graylog2, Logstash).

etc/icinga2/features-available/opentsdb.conf

@@ -0,0 +1,11 @@
+/**
+ * The OpenTsdbWriter type writes check result metrics and
+ * performance data to a OpenTSDB tcp socket.
+ */
+
+library "perfdata"
+
+object OpenTsdbWriter "opentsdb" {
+  //host = "127.0.0.1"
+  //port = 4242
+}

lib/perfdata/CMakeLists.txt

@@ -17,12 +17,13 @@
 
 mkclass_target(gelfwriter.ti gelfwriter.thpp)
 mkclass_target(graphitewriter.ti graphitewriter.thpp)
+mkclass_target(opentsdbwriter.ti opentsdbwriter.thpp)
 mkclass_target(perfdatawriter.ti perfdatawriter.thpp)
 
 mkembedconfig_target(perfdata-type.conf perfdata-type.cpp)
 
 set(perfdata_SOURCES
-  gelfwriter.cpp gelfwriter.thpp graphitewriter.cpp graphitewriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp
+  gelfwriter.cpp gelfwriter.thpp graphitewriter.cpp graphitewriter.thpp opentsdbwriter.cpp opentsdbwriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp
 )
 
 if(ICINGA2_UNITY_BUILD)
@@ -49,6 +50,11 @@ install_if_not_exists(
   ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available
 )
 
+install_if_not_exists(
+  ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/opentsdb.conf
+  ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available
+)
+
 install_if_not_exists(
   ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/perfdata.conf
   ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available

lib/perfdata/opentsdbwriter.cpp

@@ -0,0 +1,250 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org)    *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#include "perfdata/opentsdbwriter.hpp"
+#include "icinga/service.hpp"
+#include "icinga/macroprocessor.hpp"
+#include "icinga/icingaapplication.hpp"
+#include "icinga/compatutility.hpp"
+#include "icinga/perfdatavalue.hpp"
+#include "base/tcpsocket.hpp"
+#include "base/dynamictype.hpp"
+#include "base/objectlock.hpp"
+#include "base/logger.hpp"
+#include "base/convert.hpp"
+#include "base/utility.hpp"
+#include "base/application.hpp"
+#include "base/stream.hpp"
+#include "base/networkstream.hpp"
+#include "base/exception.hpp"
+#include "base/statsfunction.hpp"
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/foreach.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/replace.hpp>
+
+using namespace icinga;
+
+REGISTER_TYPE(OpenTsdbWriter);
+
+REGISTER_STATSFUNCTION(OpenTsdbWriterStats, &OpenTsdbWriter::StatsFunc);
+
+Value OpenTsdbWriter::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr&)
+{
+	Dictionary::Ptr nodes = new Dictionary();
+
+	BOOST_FOREACH(const OpenTsdbWriter::Ptr& opentsdbwriter, DynamicType::GetObjectsByType<OpenTsdbWriter>()) {
+		nodes->Set(opentsdbwriter->GetName(), 1); //add more stats
+	}
+
+	status->Set("opentsdbwriter", nodes);
+
+	return 0;
+}
+
+void OpenTsdbWriter::Start(void)
+{
+	DynamicObject::Start();
+
+	m_ReconnectTimer = new Timer();
+	m_ReconnectTimer->SetInterval(10);
+	m_ReconnectTimer->OnTimerExpired.connect(boost::bind(&OpenTsdbWriter::ReconnectTimerHandler, this));
+	m_ReconnectTimer->Start();
+	m_ReconnectTimer->Reschedule(0);
+
+	Service::OnNewCheckResult.connect(boost::bind(&OpenTsdbWriter::CheckResultHandler, this, _1, _2));
+}
+
+void OpenTsdbWriter::ReconnectTimerHandler(void)
+{
+	if (m_Stream)
+		return;
+
+	TcpSocket::Ptr socket = new TcpSocket();
+
+	Log(LogNotice, "OpenTsdbWriter")
+		<< "Reconnect to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() << "'.";
+
+	try {
+		socket->Connect(GetHost(), GetPort());
+	} catch (std::exception&) {
+		Log(LogCritical, "OpenTsdbWriter")
+			<< "Can't connect to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() << "'.";
+		return;
+	}
+
+	m_Stream = new NetworkStream(socket);
+}
+
+void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
+{
+	CONTEXT("Processing check result for '" + checkable->GetName() + "'");
+
+	if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
+		return;
+
+	Service::Ptr service = dynamic_pointer_cast<Service>(checkable);
+	Host::Ptr host;
+
+	if (service)
+		host = service->GetHost();
+	else
+		host = static_pointer_cast<Host>(checkable);
+
+	String hostName = host->GetName();
+
+	String metric;
+	std::map<String, String> tags;
+	tags["host"] = hostName;
+
+	if (service) {
+		String serviceName = service->GetShortName();
+		EscapeMetric(serviceName);
+		metric = "icinga.service." + serviceName;
+
+		SendMetric(metric + ".state", tags, service->GetState());
+	} else {
+		metric = "icinga.host";
+		SendMetric(metric + ".state", tags, host->GetState());
+	}
+
+	SendMetric(metric + ".state_type", tags, checkable->GetStateType());
+	SendMetric(metric + ".reachable", tags, checkable->IsReachable());
+	SendMetric(metric + ".downtime_depth", tags, checkable->GetDowntimeDepth());
+
+	SendPerfdata(metric, tags, cr);
+
+	metric = "icinga.check";
+
+	if (service) {
+		tags["type"] = "service";
+		String serviceName = service->GetShortName();
+		EscapeTag(serviceName);
+		tags["service"] = serviceName;
+	} else {
+		tags["type"] = "host";
+	}
+
+	SendMetric(metric + ".current_attempt", tags, checkable->GetCheckAttempt());
+	SendMetric(metric + ".max_check_attempts", tags, checkable->GetMaxCheckAttempts());
+	SendMetric(metric + ".latency", tags, Service::CalculateLatency(cr));
+	SendMetric(metric + ".execution_time", tags, Service::CalculateExecutionTime(cr));
+}
+
+void OpenTsdbWriter::SendPerfdata(const String& metric, const std::map<String, String>& tags, const CheckResult::Ptr& cr)
+{
+	Array::Ptr perfdata = cr->GetPerformanceData();
+
+	if (!perfdata)
+		return;
+
+	ObjectLock olock(perfdata);
+	BOOST_FOREACH(const Value& val, perfdata) {
+		PerfdataValue::Ptr pdv;
+
+		if (val.IsObjectType<PerfdataValue>())
+			pdv = val;
+		else {
+			try {
+				pdv = PerfdataValue::Parse(val);
+			} catch (const std::exception&) {
+				Log(LogWarning, "OpenTsdbWriter")
+					<< "Ignoring invalid perfdata value: " << val;
+				continue;
+			}
+		}
+
+		String escaped_key = pdv->GetLabel();
+		EscapeMetric(escaped_key);
+		boost::algorithm::replace_all(escaped_key, "::", ".");
+
+		SendMetric(metric + "." + escaped_key, tags, pdv->GetValue());
+
+		if (pdv->GetCrit())
+			SendMetric(metric + "." + escaped_key + "_crit", tags, pdv->GetCrit());
+		if (pdv->GetWarn())
+			SendMetric(metric + "." + escaped_key + "_warn", tags, pdv->GetWarn());
+		if (pdv->GetMin())
+			SendMetric(metric + "." + escaped_key + "_min", tags, pdv->GetMin());
+		if (pdv->GetMax())
+			SendMetric(metric + "." + escaped_key + "_max", tags, pdv->GetMax());
+	}
+}
+
+void OpenTsdbWriter::SendMetric(const String& metric, const std::map<String, String>& tags, double value)
+{
+	String tags_string = "";
+	BOOST_FOREACH(const Dictionary::Pair& tag, tags) {
+		tags_string += " " + tag.first + "=" + Convert::ToString(tag.second);
+	}
+
+	std::ostringstream msgbuf;
+	/*
+	 * must be (http://opentsdb.net/docs/build/html/user_guide/writing.html)
+	 * put <metric> <timestamp> <value> <tagk1=tagv1[ tagk2=tagv2 ...tagkN=tagvN]>
+	 * "tags" must include at least one tag, we use "host=HOSTNAME"
+	 */
+	msgbuf << "put " << metric << " " << static_cast<long>(Utility::GetTime()) << " " << Convert::ToString(value) << " " << tags_string;
+
+	Log(LogDebug, "OpenTsdbWriter")
+		<< "Add to metric list:'" << msgbuf.str() << "'.";
+
+	/* do not send \n to debug log */
+	msgbuf << "\n";
+	String put = msgbuf.str();
+
+	ObjectLock olock(this);
+
+	if (!m_Stream)
+		return;
+
+	try {
+		m_Stream->Write(put.CStr(), put.GetLength());
+	} catch (const std::exception& ex) {
+		Log(LogCritical, "OpenTsdbWriter")
+			<< "Cannot write to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() + "'.";
+
+		m_Stream.reset();
+	}
+}
+
+/* for metric and tag name rules, see
+ * http://opentsdb.net/docs/build/html/user_guide/writing.html#metrics-and-tags
+ */
+String OpenTsdbWriter::EscapeTag(const String& str)
+{
+	String result = str;
+
+	boost::replace_all(result, " ", "_");
+	boost::replace_all(result, "\\", "_");
+
+	return result;
+}
+
+String OpenTsdbWriter::EscapeMetric(const String& str)
+{
+	String result = str;
+
+	boost::replace_all(result, " ", "_");
+	boost::replace_all(result, ".", "_");
+	boost::replace_all(result, "\\", "_");
+
+	return result;
+}

lib/perfdata/opentsdbwriter.hpp

@@ -0,0 +1,65 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org)    *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#ifndef OPENTSDBWRITER_H
+#define OPENTSDBWRITER_H
+
+#include "perfdata/opentsdbwriter.thpp"
+#include "icinga/service.hpp"
+#include "base/dynamicobject.hpp"
+#include "base/tcpsocket.hpp"
+#include "base/timer.hpp"
+#include <fstream>
+
+namespace icinga
+{
+
+/**
+ * An Icinga opentsdb writer.
+ *
+ * @ingroup perfdata
+ */
+class OpenTsdbWriter : public ObjectImpl<OpenTsdbWriter>
+{
+public:
+	DECLARE_OBJECT(OpenTsdbWriter);
+	DECLARE_OBJECTNAME(OpenTsdbWriter);
+
+	static Value StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata);
+
+protected:
+	virtual void Start(void);
+
+private:
+	Stream::Ptr m_Stream;
+	
+	Timer::Ptr m_ReconnectTimer;
+
+	void CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
+	void SendMetric(const String& metric, const std::map<String, String>& tags, double value);
+	void SendPerfdata(const String& metric, const std::map<String, String>& tags, const CheckResult::Ptr& cr);
+	static String EscapeTag(const String& str);
+	static String EscapeMetric(const String& str);
+
+	void ReconnectTimerHandler(void);
+};
+
+}
+
+#endif /* OPENTSDBWRITER_H */

lib/perfdata/opentsdbwriter.ti

@@ -0,0 +1,16 @@
+#include "base/dynamicobject.hpp"
+
+namespace icinga
+{
+
+class OpenTsdbWriter : DynamicObject
+{
+	[config] String host {
+		default {{{ return "127.0.0.1"; }}}
+	};
+	[config] String port {
+		default {{{ return "4242"; }}}
+	};
+};
+
+}

lib/perfdata/perfdata-type.conf

@@ -40,3 +40,8 @@
 	%attribute %string "source"
 }
 
+%type OpenTsdbWriter {
+	%attribute %string "host",
+	%attribute %string "port",
+}
+