diff --git a/doc/3-monitoring-basics.md b/doc/3-monitoring-basics.md index eafecb5f6..4df47d8a5 100644 --- a/doc/3-monitoring-basics.md +++ b/doc/3-monitoring-basics.md @@ -2362,6 +2362,69 @@ Currently these events are processed: * State changes * Notifications +### OpenTSDB Writer + +While there are some OpenTSDB collector scripts and daemons like tcollector available for +Icinga 1.x it's more reasonable to directly process the check and plugin performance +in memory in Icinga 2. Once there are new metrics available, Icinga 2 will directly +write them to the defined TSDB TCP socket. + +You can enable the feature using + + # icinga2 feature enable opentsdb + +By default the `OpenTsdbWriter` object expects the TSD to listen at +`127.0.0.1` on port `4242`. + +The current naming schema is + + icinga.host. + icinga.service.. + +for host and service checks. The tag host is always applied. + +To make sure Icinga 2 writes a valid metric into OpenTSDB some characters are replaced +with `_` in the target name: + + \ (and space) + +The resulting name in OpenTSDB might look like: + + www-01 / http-cert / response time + icinga.http_cert.response_time + +In addition to the performance data retrieved from the check plugin, Icinga 2 sends +internal check statistic data to OpenTSDB: + + metric | description + -------------------|------------------------------------------ + current_attempt | current check attempt + max_check_attempts | maximum check attempts until the hard state is reached + reachable | checked object is reachable + downtime_depth | number of downtimes this object is in + execution_time | check execution time + latency | check latency + state | current state of the checked object + state_type | 0=SOFT, 1=HARD state + +While reachable, state and state_type are metrics for the host or service the +other metrics follow the current naming schema + + icinga.check. + +with the following tags + + tag | description + --------|------------------------------------------ + type | the check type, one of [host, service] + host | hostname, the check ran on + service | the service name (if type=service) + +> **Note** +> +> You might want to set the tsd.core.auto_create_metrics setting to `true` +> in your opentsdb.conf configuration file. + ## Status Data diff --git a/doc/5-object-types.md b/doc/5-object-types.md index 7d14d429c..0cff0c405 100644 --- a/doc/5-object-types.md +++ b/doc/5-object-types.md @@ -744,6 +744,27 @@ Example with your custom [global constant](15-language-reference.md#constants) ` host_name_template = GraphiteEnv + ".$host.name$" service_name_template = GraphiteEnv + ".$host.name$.$service.name$" +## OpenTsdbWriter + +Writes check result metrics and performance data to OpenTSDB. + +Example: + + library "perfdata" + + object OpenTsdbWriter "opentsdb" { + host = "127.0.0.1" + port = 4242 + } + +Attributes: + + Name |Description + ----------------------|---------------------- + host |**Optional.** OpenTSDB host address. Defaults to '127.0.0.1'. + port |**Optional.** OpenTSDB port. Defaults to 4242. + + ## GelfWriter Writes event log entries to a defined GELF receiver host (Graylog2, Logstash). diff --git a/etc/icinga2/features-available/opentsdb.conf b/etc/icinga2/features-available/opentsdb.conf new file mode 100644 index 000000000..fcb547d01 --- /dev/null +++ b/etc/icinga2/features-available/opentsdb.conf @@ -0,0 +1,11 @@ +/** + * The OpenTsdbWriter type writes check result metrics and + * performance data to a OpenTSDB tcp socket. + */ + +library "perfdata" + +object OpenTsdbWriter "opentsdb" { + //host = "127.0.0.1" + //port = 4242 +} diff --git a/lib/perfdata/CMakeLists.txt b/lib/perfdata/CMakeLists.txt index ababe1819..ff603f788 100644 --- a/lib/perfdata/CMakeLists.txt +++ b/lib/perfdata/CMakeLists.txt @@ -17,12 +17,13 @@ mkclass_target(gelfwriter.ti gelfwriter.thpp) mkclass_target(graphitewriter.ti graphitewriter.thpp) +mkclass_target(opentsdbwriter.ti opentsdbwriter.thpp) mkclass_target(perfdatawriter.ti perfdatawriter.thpp) mkembedconfig_target(perfdata-type.conf perfdata-type.cpp) set(perfdata_SOURCES - gelfwriter.cpp gelfwriter.thpp graphitewriter.cpp graphitewriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp + gelfwriter.cpp gelfwriter.thpp graphitewriter.cpp graphitewriter.thpp opentsdbwriter.cpp opentsdbwriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp ) if(ICINGA2_UNITY_BUILD) @@ -49,6 +50,11 @@ install_if_not_exists( ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available ) +install_if_not_exists( + ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/opentsdb.conf + ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available +) + install_if_not_exists( ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/perfdata.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp new file mode 100644 index 000000000..543116d4b --- /dev/null +++ b/lib/perfdata/opentsdbwriter.cpp @@ -0,0 +1,250 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "perfdata/opentsdbwriter.hpp" +#include "icinga/service.hpp" +#include "icinga/macroprocessor.hpp" +#include "icinga/icingaapplication.hpp" +#include "icinga/compatutility.hpp" +#include "icinga/perfdatavalue.hpp" +#include "base/tcpsocket.hpp" +#include "base/dynamictype.hpp" +#include "base/objectlock.hpp" +#include "base/logger.hpp" +#include "base/convert.hpp" +#include "base/utility.hpp" +#include "base/application.hpp" +#include "base/stream.hpp" +#include "base/networkstream.hpp" +#include "base/exception.hpp" +#include "base/statsfunction.hpp" +#include +#include +#include +#include +#include + +using namespace icinga; + +REGISTER_TYPE(OpenTsdbWriter); + +REGISTER_STATSFUNCTION(OpenTsdbWriterStats, &OpenTsdbWriter::StatsFunc); + +Value OpenTsdbWriter::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr&) +{ + Dictionary::Ptr nodes = new Dictionary(); + + BOOST_FOREACH(const OpenTsdbWriter::Ptr& opentsdbwriter, DynamicType::GetObjectsByType()) { + nodes->Set(opentsdbwriter->GetName(), 1); //add more stats + } + + status->Set("opentsdbwriter", nodes); + + return 0; +} + +void OpenTsdbWriter::Start(void) +{ + DynamicObject::Start(); + + m_ReconnectTimer = new Timer(); + m_ReconnectTimer->SetInterval(10); + m_ReconnectTimer->OnTimerExpired.connect(boost::bind(&OpenTsdbWriter::ReconnectTimerHandler, this)); + m_ReconnectTimer->Start(); + m_ReconnectTimer->Reschedule(0); + + Service::OnNewCheckResult.connect(boost::bind(&OpenTsdbWriter::CheckResultHandler, this, _1, _2)); +} + +void OpenTsdbWriter::ReconnectTimerHandler(void) +{ + if (m_Stream) + return; + + TcpSocket::Ptr socket = new TcpSocket(); + + Log(LogNotice, "OpenTsdbWriter") + << "Reconnect to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() << "'."; + + try { + socket->Connect(GetHost(), GetPort()); + } catch (std::exception&) { + Log(LogCritical, "OpenTsdbWriter") + << "Can't connect to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() << "'."; + return; + } + + m_Stream = new NetworkStream(socket); +} + +void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr) +{ + CONTEXT("Processing check result for '" + checkable->GetName() + "'"); + + if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata()) + return; + + Service::Ptr service = dynamic_pointer_cast(checkable); + Host::Ptr host; + + if (service) + host = service->GetHost(); + else + host = static_pointer_cast(checkable); + + String hostName = host->GetName(); + + String metric; + std::map tags; + tags["host"] = hostName; + + if (service) { + String serviceName = service->GetShortName(); + EscapeMetric(serviceName); + metric = "icinga.service." + serviceName; + + SendMetric(metric + ".state", tags, service->GetState()); + } else { + metric = "icinga.host"; + SendMetric(metric + ".state", tags, host->GetState()); + } + + SendMetric(metric + ".state_type", tags, checkable->GetStateType()); + SendMetric(metric + ".reachable", tags, checkable->IsReachable()); + SendMetric(metric + ".downtime_depth", tags, checkable->GetDowntimeDepth()); + + SendPerfdata(metric, tags, cr); + + metric = "icinga.check"; + + if (service) { + tags["type"] = "service"; + String serviceName = service->GetShortName(); + EscapeTag(serviceName); + tags["service"] = serviceName; + } else { + tags["type"] = "host"; + } + + SendMetric(metric + ".current_attempt", tags, checkable->GetCheckAttempt()); + SendMetric(metric + ".max_check_attempts", tags, checkable->GetMaxCheckAttempts()); + SendMetric(metric + ".latency", tags, Service::CalculateLatency(cr)); + SendMetric(metric + ".execution_time", tags, Service::CalculateExecutionTime(cr)); +} + +void OpenTsdbWriter::SendPerfdata(const String& metric, const std::map& tags, const CheckResult::Ptr& cr) +{ + Array::Ptr perfdata = cr->GetPerformanceData(); + + if (!perfdata) + return; + + ObjectLock olock(perfdata); + BOOST_FOREACH(const Value& val, perfdata) { + PerfdataValue::Ptr pdv; + + if (val.IsObjectType()) + pdv = val; + else { + try { + pdv = PerfdataValue::Parse(val); + } catch (const std::exception&) { + Log(LogWarning, "OpenTsdbWriter") + << "Ignoring invalid perfdata value: " << val; + continue; + } + } + + String escaped_key = pdv->GetLabel(); + EscapeMetric(escaped_key); + boost::algorithm::replace_all(escaped_key, "::", "."); + + SendMetric(metric + "." + escaped_key, tags, pdv->GetValue()); + + if (pdv->GetCrit()) + SendMetric(metric + "." + escaped_key + "_crit", tags, pdv->GetCrit()); + if (pdv->GetWarn()) + SendMetric(metric + "." + escaped_key + "_warn", tags, pdv->GetWarn()); + if (pdv->GetMin()) + SendMetric(metric + "." + escaped_key + "_min", tags, pdv->GetMin()); + if (pdv->GetMax()) + SendMetric(metric + "." + escaped_key + "_max", tags, pdv->GetMax()); + } +} + +void OpenTsdbWriter::SendMetric(const String& metric, const std::map& tags, double value) +{ + String tags_string = ""; + BOOST_FOREACH(const Dictionary::Pair& tag, tags) { + tags_string += " " + tag.first + "=" + Convert::ToString(tag.second); + } + + std::ostringstream msgbuf; + /* + * must be (http://opentsdb.net/docs/build/html/user_guide/writing.html) + * put + * "tags" must include at least one tag, we use "host=HOSTNAME" + */ + msgbuf << "put " << metric << " " << static_cast(Utility::GetTime()) << " " << Convert::ToString(value) << " " << tags_string; + + Log(LogDebug, "OpenTsdbWriter") + << "Add to metric list:'" << msgbuf.str() << "'."; + + /* do not send \n to debug log */ + msgbuf << "\n"; + String put = msgbuf.str(); + + ObjectLock olock(this); + + if (!m_Stream) + return; + + try { + m_Stream->Write(put.CStr(), put.GetLength()); + } catch (const std::exception& ex) { + Log(LogCritical, "OpenTsdbWriter") + << "Cannot write to OpenTSDB TSD on host '" << GetHost() << "' port '" << GetPort() + "'."; + + m_Stream.reset(); + } +} + +/* for metric and tag name rules, see + * http://opentsdb.net/docs/build/html/user_guide/writing.html#metrics-and-tags + */ +String OpenTsdbWriter::EscapeTag(const String& str) +{ + String result = str; + + boost::replace_all(result, " ", "_"); + boost::replace_all(result, "\\", "_"); + + return result; +} + +String OpenTsdbWriter::EscapeMetric(const String& str) +{ + String result = str; + + boost::replace_all(result, " ", "_"); + boost::replace_all(result, ".", "_"); + boost::replace_all(result, "\\", "_"); + + return result; +} diff --git a/lib/perfdata/opentsdbwriter.hpp b/lib/perfdata/opentsdbwriter.hpp new file mode 100644 index 000000000..1f6ada102 --- /dev/null +++ b/lib/perfdata/opentsdbwriter.hpp @@ -0,0 +1,65 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef OPENTSDBWRITER_H +#define OPENTSDBWRITER_H + +#include "perfdata/opentsdbwriter.thpp" +#include "icinga/service.hpp" +#include "base/dynamicobject.hpp" +#include "base/tcpsocket.hpp" +#include "base/timer.hpp" +#include + +namespace icinga +{ + +/** + * An Icinga opentsdb writer. + * + * @ingroup perfdata + */ +class OpenTsdbWriter : public ObjectImpl +{ +public: + DECLARE_OBJECT(OpenTsdbWriter); + DECLARE_OBJECTNAME(OpenTsdbWriter); + + static Value StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata); + +protected: + virtual void Start(void); + +private: + Stream::Ptr m_Stream; + + Timer::Ptr m_ReconnectTimer; + + void CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr); + void SendMetric(const String& metric, const std::map& tags, double value); + void SendPerfdata(const String& metric, const std::map& tags, const CheckResult::Ptr& cr); + static String EscapeTag(const String& str); + static String EscapeMetric(const String& str); + + void ReconnectTimerHandler(void); +}; + +} + +#endif /* OPENTSDBWRITER_H */ diff --git a/lib/perfdata/opentsdbwriter.ti b/lib/perfdata/opentsdbwriter.ti new file mode 100644 index 000000000..4a21f2943 --- /dev/null +++ b/lib/perfdata/opentsdbwriter.ti @@ -0,0 +1,16 @@ +#include "base/dynamicobject.hpp" + +namespace icinga +{ + +class OpenTsdbWriter : DynamicObject +{ + [config] String host { + default {{{ return "127.0.0.1"; }}} + }; + [config] String port { + default {{{ return "4242"; }}} + }; +}; + +} diff --git a/lib/perfdata/perfdata-type.conf b/lib/perfdata/perfdata-type.conf index 99245b253..42938b4f1 100644 --- a/lib/perfdata/perfdata-type.conf +++ b/lib/perfdata/perfdata-type.conf @@ -40,3 +40,8 @@ %attribute %string "source" } +%type OpenTsdbWriter { + %attribute %string "host", + %attribute %string "port", +} +