From 3cd6b66cd430a4f080a6aeabca5b9e4ef2ac21ca Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Tue, 23 Jul 2019 23:28:42 +1000 Subject: [PATCH 1/7] Initial commit, add custom tag functionality --- lib/perfdata/opentsdbwriter.cpp | 71 ++++++++++++++++++++++++++++++++- lib/perfdata/opentsdbwriter.hpp | 3 ++ lib/perfdata/opentsdbwriter.ti | 20 ++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index 38405a2a3..527acf4c9 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -151,15 +151,54 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C Service::Ptr service = dynamic_pointer_cast(checkable); Host::Ptr host; + Dictionary::Ptr config_tmpl_clean; - if (service) + if (service) { host = service->GetHost(); - else + config_tmpl_clean = GetServiceTemplate(); + } + else { host = static_pointer_cast(checkable); + config_tmpl_clean = GetHostTemplate(); + } + + // Clone the config template and perform an in-place macro expansion of measurement and tag values + Dictionary::Ptr config_tmpl = static_pointer_cast(config_tmpl_clean->Clone()); + Dictionary::Ptr config_tmpl_tags = config_tmpl->Get("tags"); + + // Configure config template macro resolver + MacroProcessor::ResolverList resolvers; + if (service) + resolvers.emplace_back("service", service); + resolvers.emplace_back("host", host); + resolvers.emplace_back("icinga", IcingaApplication::GetInstance()); String metric; std::map tags; + // Resolve macros in configuration template + if (config_tmpl_tags) { + + ObjectLock olock(config_tmpl_tags); + + for (const Dictionary::Pair& pair : config_tmpl_tags) { + + String missing_macro; + Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro); + + if (!missing_macro.IsEmpty()) { + Log(LogDebug, "OpenTsdbWriter") + << "Macro in config template does not exist:'" << missing_macro << "'."; + + continue; + } + + String tagname = pair.first; + tags[tagname] = EscapeTag(value); + + } + } + String escaped_hostName = EscapeTag(host->GetName()); tags["host"] = escaped_hostName; @@ -338,3 +377,31 @@ String OpenTsdbWriter::EscapeMetric(const String& str) return result; } + +void OpenTsdbWriter::ValidateHostTemplate(const Lazy& lvalue, const ValidationUtils& utils) +{ + ObjectImpl::ValidateHostTemplate(lvalue, utils); + + Dictionary::Ptr tags = lvalue()->Get("tags"); + if (tags) { + ObjectLock olock(tags); + for (const Dictionary::Pair& pair : tags) { + if (!MacroProcessor::ValidateMacroString(pair.second)) + BOOST_THROW_EXCEPTION(ValidationError(this, { "host_template", "tags", pair.first }, "Closing $ not found in macro format string '" + pair.second)); + } + } +} + +void OpenTsdbWriter::ValidateServiceTemplate(const Lazy& lvalue, const ValidationUtils& utils) +{ + ObjectImpl::ValidateServiceTemplate(lvalue, utils); + + Dictionary::Ptr tags = lvalue()->Get("tags"); + if (tags) { + ObjectLock olock(tags); + for (const Dictionary::Pair& pair : tags) { + if (!MacroProcessor::ValidateMacroString(pair.second)) + BOOST_THROW_EXCEPTION(ValidationError(this, { "service_template", "tags", pair.first }, "Closing $ not found in macro format string '" + pair.second)); + } + } +} \ No newline at end of file diff --git a/lib/perfdata/opentsdbwriter.hpp b/lib/perfdata/opentsdbwriter.hpp index 5aff981cd..4dd98811e 100644 --- a/lib/perfdata/opentsdbwriter.hpp +++ b/lib/perfdata/opentsdbwriter.hpp @@ -26,6 +26,9 @@ public: static void StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata); + void ValidateHostTemplate(const Lazy& lvalue, const ValidationUtils& utils) override; + void ValidateServiceTemplate(const Lazy& lvalue, const ValidationUtils& utils) override; + protected: void OnConfigLoaded() override; void Resume() override; diff --git a/lib/perfdata/opentsdbwriter.ti b/lib/perfdata/opentsdbwriter.ti index de19a1eac..7a0219f01 100644 --- a/lib/perfdata/opentsdbwriter.ti +++ b/lib/perfdata/opentsdbwriter.ti @@ -20,6 +20,13 @@ class OpenTsdbWriter : ConfigObject [config] bool enable_ha { default {{{ return false; }}} }; + [config] Dictionary::Ptr host_template { + default {{{ return new Dictionary(); }}} + + }; + [config] Dictionary::Ptr service_template { + default {{{ return new Dictionary(); }}} + }; [no_user_modify] bool connected; [no_user_modify] bool should_connect { @@ -27,4 +34,17 @@ class OpenTsdbWriter : ConfigObject }; }; +validator OpenTsdbWriter { + Dictionary host_template { + Dictionary "tags" { + String "*"; + }; + }; + Dictionary service_template { + Dictionary "tags" { + String "*"; + }; + }; +}; + } From 9c5bbdfe10d67c66535409562b1acafe6a403780 Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Wed, 24 Jul 2019 00:25:35 +1000 Subject: [PATCH 2/7] Added OpenTSDB custom tagging documentation Added OpenTSDB custom tagging configuration template --- doc/09-object-types.md | 2 + doc/14-features.md | 90 ++++++++++++++++++++ etc/icinga2/features-available/opentsdb.conf | 13 +++ 3 files changed, 105 insertions(+) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index 40f161bc4..f0d208030 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1703,6 +1703,8 @@ Configuration Attributes: host | String | **Optional.** OpenTSDB host address. Defaults to `127.0.0.1`. port | Number | **Optional.** OpenTSDB port. Defaults to `4242`. enable\_ha | Boolean | **Optional.** Enable the high availability functionality. Only valid in a [cluster setup](06-distributed-monitoring.md#distributed-monitoring-high-availability-features). Defaults to `false`. + host_template | Dictionary | **Optional.** Specify additional tags to be included with host metrics. This requires a sub-dictionary named `tags`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`. + service_template | Dictionary | **Optional.** Specify additional tags to be included with service metrics. This requires a sub-dictionary named `tags`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`. ### PerfdataWriter diff --git a/doc/14-features.md b/doc/14-features.md index c119abd91..165a45319 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -652,6 +652,96 @@ with the following tags > You might want to set the tsd.core.auto_create_metrics setting to `true` > in your opentsdb.conf configuration file. +#### Custom Tags + +In addition to the default tags listed above, it is possible to send +your own custom tags with your data to OpenTSDB. + +Note that custom tags are sent **in addition** to the default hostname, +type and service name tags. If you do not include this section in the +config file, no custom tags will be included. + +Custom tags can be custom attributes or built in attributes. + +Consider a host object: + +``` +object Host "my-server1" { + address = "10.0.0.1" + check_command = "hostalive" + vars.location = "Australia" +} +``` + +and a service object: + +``` +object Service "ping" { + host_name = "localhost" + check_command = "my-ping" + + vars.ping_packets = 10 +} +``` + +It is possible to send `vars.location` and `vars.ping_packets` along +with performance data. Additionally, any other attribute can be sent +as a tag, such as `check_command`. + +You can make use of the `host_template` and `service_template` blocks +in the `opentsdb.conf` configuration file. + +An example OpenTSDB configuration file which makes use of custom tags: + +``` +object OpenTsdbWriter "opentsdb" { + host = "127.0.0.1" + port = 4242 + host_template = { + tags = { + location = "$host.vars.location$" + checkcommand = "$host.check_command$" + } + } + service_template = { + tags = { + location = "$host.vars.location$" + pingpackets = "$service.vars.ping_packets$" + checkcommand = "$service.check_command$" + } + } +} +``` + +Depending on what keyword the macro begins with, will determine what +attributes are available in the macro context. The below table explains +what attributes are available with links to each object type. + + start of macro | description + ---------------|------------------------------------------ + \$host...$ | Attributes available on a [Host object](09-object-types.md#objecttype-host) + \$service...$ | Attributes available on a [Service object](09-object-types.md#objecttype-service) + \$icinga...$ | Attributes available on the [IcingaApplication object](09-object-types.md#icingaapplication) + +> **Note** +> +> Ensure you do not name your custom attributes with a dot in the name. +> Dots located inside a macro tell the interpreter to expand a +> dictionary. +> +> Do not do this in your object configuration: +> +> `vars["my.attribute"]` +> +> as you will be unable to reference `my.attribute` because it is not a +> dictionary. +> +> Instead, use underscores or another character: +> +> `vars.my_attribute` or `vars["my_attribute"]` + + + #### OpenTSDB in Cluster HA Zones The OpenTSDB feature supports [high availability](06-distributed-monitoring.md#distributed-monitoring-high-availability-features) diff --git a/etc/icinga2/features-available/opentsdb.conf b/etc/icinga2/features-available/opentsdb.conf index 540556c10..a98141daf 100644 --- a/etc/icinga2/features-available/opentsdb.conf +++ b/etc/icinga2/features-available/opentsdb.conf @@ -6,4 +6,17 @@ object OpenTsdbWriter "opentsdb" { //host = "127.0.0.1" //port = 4242 + + // Custom Tagging, refer to Icinga object type documentation for + // OpenTsdbWriter + //host_template = { + // tags = { + // checkcommand = "$host.check_command$" + // } + //} + //service_template = { + // tags = { + // checkcommand = "$service.check_command$" + // } + //} } From 90c42e1bbc889708822d320b8815f86cc61490f5 Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Wed, 24 Jul 2019 00:28:27 +1000 Subject: [PATCH 3/7] Re-worded debug log message --- lib/perfdata/opentsdbwriter.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index 527acf4c9..f3538637f 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -176,7 +176,7 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C String metric; std::map tags; - // Resolve macros in configuration template + // Resolve macros in configuration template and build custom tag list if (config_tmpl_tags) { ObjectLock olock(config_tmpl_tags); @@ -188,7 +188,8 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C if (!missing_macro.IsEmpty()) { Log(LogDebug, "OpenTsdbWriter") - << "Macro in config template does not exist:'" << missing_macro << "'."; + << "Unable to resolve macro:'" << missing_macro + << "' for this host or service."; continue; } From 619a487733d1cedb8ad9a304b473971ae976ed49 Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Wed, 24 Jul 2019 00:40:56 +1000 Subject: [PATCH 4/7] Added validation function comments --- lib/perfdata/opentsdbwriter.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index f3538637f..30de4c46f 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -379,6 +379,13 @@ String OpenTsdbWriter::EscapeMetric(const String& str) return result; } +/** +* Validates the host_template configuration block in the configuration +* file and checks for syntax errors. +* +* @param lvalue The host_template dictionary +* @param utils Validation helper utilities +*/ void OpenTsdbWriter::ValidateHostTemplate(const Lazy& lvalue, const ValidationUtils& utils) { ObjectImpl::ValidateHostTemplate(lvalue, utils); @@ -393,6 +400,13 @@ void OpenTsdbWriter::ValidateHostTemplate(const Lazy& lvalue, c } } +/** +* Validates the service_template configuration block in the +* configuration file and checks for syntax errors. +* +* @param lvalue The service_template dictionary +* @param utils Validation helper utilities +*/ void OpenTsdbWriter::ValidateServiceTemplate(const Lazy& lvalue, const ValidationUtils& utils) { ObjectImpl::ValidateServiceTemplate(lvalue, utils); From c2c4b97ea50d09534e2a6bf3b2bd62852ab9bbfa Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Wed, 24 Jul 2019 00:42:53 +1000 Subject: [PATCH 5/7] Adjusted explicit String cast --- lib/perfdata/opentsdbwriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index 30de4c46f..1f0f1e5be 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -194,7 +194,7 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C continue; } - String tagname = pair.first; + String tagname = Convert::ToString(pair.first); tags[tagname] = EscapeTag(value); } From 0b8fdfd7f2f61db09c355eddf123f76ee4aef18f Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Sat, 27 Jul 2019 01:31:16 +1000 Subject: [PATCH 6/7] Removed dictionary cloning behaviour. Added configuration caching on instantiation. --- lib/perfdata/opentsdbwriter.cpp | 63 ++++++++++++++++++++++++++------- lib/perfdata/opentsdbwriter.hpp | 6 ++++ 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index 1f0f1e5be..5b90ed52c 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -73,6 +73,8 @@ void OpenTsdbWriter::Resume() Log(LogInformation, "OpentsdbWriter") << "'" << GetName() << "' resumed."; + ReadConfigTemplate(m_ServiceConfigTemplate, m_HostConfigTemplate); + m_ReconnectTimer = new Timer(); m_ReconnectTimer->SetInterval(10); m_ReconnectTimer->OnTimerExpired.connect(std::bind(&OpenTsdbWriter::ReconnectTimerHandler, this)); @@ -151,33 +153,35 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C Service::Ptr service = dynamic_pointer_cast(checkable); Host::Ptr host; - Dictionary::Ptr config_tmpl_clean; + Dictionary::Ptr config_tmpl; + Dictionary::Ptr config_tmpl_tags; if (service) { host = service->GetHost(); - config_tmpl_clean = GetServiceTemplate(); + config_tmpl = m_ServiceConfigTemplate; } else { host = static_pointer_cast(checkable); - config_tmpl_clean = GetHostTemplate(); + config_tmpl = m_HostConfigTemplate; } - // Clone the config template and perform an in-place macro expansion of measurement and tag values - Dictionary::Ptr config_tmpl = static_pointer_cast(config_tmpl_clean->Clone()); - Dictionary::Ptr config_tmpl_tags = config_tmpl->Get("tags"); - - // Configure config template macro resolver - MacroProcessor::ResolverList resolvers; - if (service) - resolvers.emplace_back("service", service); - resolvers.emplace_back("host", host); - resolvers.emplace_back("icinga", IcingaApplication::GetInstance()); + // Get the tags nested dictionary in the service/host template in the config + if (config_tmpl) { + config_tmpl_tags = config_tmpl->Get("tags"); + } String metric; std::map tags; // Resolve macros in configuration template and build custom tag list if (config_tmpl_tags) { + + // Configure config template macro resolver + MacroProcessor::ResolverList resolvers; + if (service) + resolvers.emplace_back("service", service); + resolvers.emplace_back("host", host); + resolvers.emplace_back("icinga", IcingaApplication::GetInstance()); ObjectLock olock(config_tmpl_tags); @@ -379,6 +383,39 @@ String OpenTsdbWriter::EscapeMetric(const String& str) return result; } +/** +* Saves the template dictionaries defined in the config file into running memory +* +* @param stemplate The dictionary to save the service configuration to +* @param htemplate The dictionary to save the host configuration to +*/ +void OpenTsdbWriter::ReadConfigTemplate(const Dictionary::Ptr& stemplate, + const Dictionary::Ptr& htemplate) +{ + + m_ServiceConfigTemplate = GetServiceTemplate(); + + if (!m_ServiceConfigTemplate) { + Log(LogDebug, "OpenTsdbWriter") + << "Unable to locate service template configuration."; + } else if (m_ServiceConfigTemplate->GetLength() == 0) { + Log(LogDebug, "OpenTsdbWriter") + << "The service template configuration is empty."; + } + + m_HostConfigTemplate = GetHostTemplate(); + + if (!m_HostConfigTemplate) { + Log(LogDebug, "OpenTsdbWriter") + << "Unable to locate host template configuration."; + } else if (m_HostConfigTemplate->GetLength() == 0) { + Log(LogDebug, "OpenTsdbWriter") + << "The host template configuration is empty."; + } + +} + + /** * Validates the host_template configuration block in the configuration * file and checks for syntax errors. diff --git a/lib/perfdata/opentsdbwriter.hpp b/lib/perfdata/opentsdbwriter.hpp index 4dd98811e..5531bdae2 100644 --- a/lib/perfdata/opentsdbwriter.hpp +++ b/lib/perfdata/opentsdbwriter.hpp @@ -39,6 +39,9 @@ private: Timer::Ptr m_ReconnectTimer; + Dictionary::Ptr m_ServiceConfigTemplate; + Dictionary::Ptr m_HostConfigTemplate; + void CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr); void SendMetric(const Checkable::Ptr& checkable, const String& metric, const std::map& tags, double value, double ts); @@ -48,6 +51,9 @@ private: static String EscapeMetric(const String& str); void ReconnectTimerHandler(); + + void ReadConfigTemplate(const Dictionary::Ptr& stemplate, + const Dictionary::Ptr& htemplate); }; } From 166e79fee1102a8a2dd746e4d7edf6b74addae1b Mon Sep 17 00:00:00 2001 From: Ant1x <37016240+Ant1x@users.noreply.github.com> Date: Wed, 23 Oct 2019 23:58:27 +1100 Subject: [PATCH 7/7] Added OpenTSDB Metric prefix naming support Added OpenTSDB Generic Metric functionality --- doc/09-object-types.md | 5 +- doc/14-features.md | 94 ++++++++++++++++++-- etc/icinga2/features-available/opentsdb.conf | 7 +- lib/perfdata/opentsdbwriter.cpp | 91 +++++++++++++++---- lib/perfdata/opentsdbwriter.ti | 5 ++ 5 files changed, 175 insertions(+), 27 deletions(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index f0d208030..9c8a73baf 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1703,8 +1703,9 @@ Configuration Attributes: host | String | **Optional.** OpenTSDB host address. Defaults to `127.0.0.1`. port | Number | **Optional.** OpenTSDB port. Defaults to `4242`. enable\_ha | Boolean | **Optional.** Enable the high availability functionality. Only valid in a [cluster setup](06-distributed-monitoring.md#distributed-monitoring-high-availability-features). Defaults to `false`. - host_template | Dictionary | **Optional.** Specify additional tags to be included with host metrics. This requires a sub-dictionary named `tags`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`. - service_template | Dictionary | **Optional.** Specify additional tags to be included with service metrics. This requires a sub-dictionary named `tags`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`. + enable_generic_metrics | Boolean | **Optional.** Re-use metric names to store different perfdata values for a particular check. Use tags to distinguish perfdata instead of metric name. Defaults to `false`. + host_template | Dictionary | **Optional.** Specify additional tags to be included with host metrics. This requires a sub-dictionary named `tags`. Also specify a naming prefix by setting `metric`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags) and [OpenTSDB Metric Prefix](14-features.md#opentsdb-metric-prefix). More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags). Defaults to an `empty Dictionary`. + service_template | Dictionary | **Optional.** Specify additional tags to be included with service metrics. This requires a sub-dictionary named `tags`. Also specify a naming prefix by setting `metric`. More information can be found in [OpenTSDB custom tags](14-features.md#opentsdb-custom-tags) and [OpenTSDB Metric Prefix](14-features.md#opentsdb-metric-prefix). Defaults to an `empty Dictionary`. ### PerfdataWriter diff --git a/doc/14-features.md b/doc/14-features.md index 165a45319..d3fe25165 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -594,20 +594,33 @@ You can enable the feature using By default the `OpenTsdbWriter` object expects the TSD to listen at `127.0.0.1` on port `4242`. -The current naming schema is +The current default naming schema is: ``` -icinga.host. -icinga.service.. +icinga.host. +icinga.service.. ``` -for host and service checks. The tag host is always applied. +for host and service checks. The tag `host` is always applied. + +Icinga also sends perfdata warning, critical, minimum and maximum threshold values to OpenTSDB. +These are stored as new OpenTSDB metric names appended with `_warn`, `_crit`, `_min`, `_max`. +Values are only stored when the corresponding threshold exists in Icinga's perfdata. + +Example: +``` +icinga.service.. +icinga.service..._warn +icinga.service..._crit +icinga.service..._min +icinga.service..._max +``` To make sure Icinga 2 writes a valid metric into OpenTSDB some characters are replaced with `_` in the target name: ``` -\ (and space) +\ : (and space) ``` The resulting name in OpenTSDB might look like: @@ -652,6 +665,77 @@ with the following tags > You might want to set the tsd.core.auto_create_metrics setting to `true` > in your opentsdb.conf configuration file. +#### OpenTSDB Metric Prefix +Functionality exists to modify the built in OpenTSDB metric names that the plugin +writes to. By default this is `icinga.host` and `icinga.service.`. + +These prefixes can be modified as necessary to any arbitary string. The prefix +configuration also supports Icinga macros, so if you rather use `` +or any other variable instead of `` you may do so. + +To configure OpenTSDB metric name prefixes, create or modify the `host_template` and/or +`service_template` blocks in the `opentsdb.conf` file, to add a `metric` definition. +These modifications go hand in hand with the **OpenTSDB Custom Tag Support** detailed below, +and more information around macro use can be found there. + +Additionally, using custom Metric Prefixes or your own macros in the prefix may be +helpful if you are using the **OpenTSDB Generic Metric** functionality detailed below. + +An example configuration which includes prefix name modification: + +``` +object OpenTsdbWriter "opentsdb" { + host = "127.0.0.1" + port = 4242 + host_template = { + metric = "icinga.myhost" + tags = { + location = "$host.vars.location$" + checkcommand = "$host.check_command$" + } + } + service_template = { + metric = "icinga.service.$service.check_command$" + } +} +``` + +The above configuration will output the following naming schema: +``` +icinga.myhost. +icinga.service.. +``` +Note how `` is always appended in the default naming schema mode. + +#### OpenTSDB Generic Metric Naming Schema + +An alternate naming schema (`Generic Metrics`) is available where OpenTSDB metric names are more generic +and do not include the Icinga perfdata label in the metric name. Instead, +perfdata labels are stored in a tag `label` which is stored along with each perfdata value. + +This ultimately reduces the number of unique OpenTSDB metric names which may make +querying aggregate data easier. This also allows you to store all perfdata values for a +particular check inside one OpenTSDB metric name for each check. + +This alternate naming schema can be enabled by setting the following in the OpenTSDBWriter config: +`enable_generic_metrics = true` + +> **Tip** +> Consider using `Generic Metrics` along with the **OpenTSDB Metric Prefix** naming options +> described above + +An example of this naming schema when compared to the default is: + +``` +icinga.host +icinga.service. +``` + +> **Note** +> Note how `` does not appear in the OpenTSDB metric name +> when using `Generic Metrics`. Instead, a new tag `label` appears on each value written +> to OpenTSDB which contains the perfdata label. + #### Custom Tags In addition to the default tags listed above, it is possible to send diff --git a/etc/icinga2/features-available/opentsdb.conf b/etc/icinga2/features-available/opentsdb.conf index a98141daf..471a62268 100644 --- a/etc/icinga2/features-available/opentsdb.conf +++ b/etc/icinga2/features-available/opentsdb.conf @@ -6,17 +6,20 @@ object OpenTsdbWriter "opentsdb" { //host = "127.0.0.1" //port = 4242 + //enable_generic_metrics = false // Custom Tagging, refer to Icinga object type documentation for // OpenTsdbWriter //host_template = { + // metric = "icinga.host" // tags = { - // checkcommand = "$host.check_command$" + // zone = "$host.zone$" // } //} //service_template = { + // metric = "icinga.service.$service.check_command$" // tags = { - // checkcommand = "$service.check_command$" + // zone = "$service.zone$" // } //} } diff --git a/lib/perfdata/opentsdbwriter.cpp b/lib/perfdata/opentsdbwriter.cpp index 5b90ed52c..125ecaaed 100644 --- a/lib/perfdata/opentsdbwriter.cpp +++ b/lib/perfdata/opentsdbwriter.cpp @@ -155,6 +155,7 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C Host::Ptr host; Dictionary::Ptr config_tmpl; Dictionary::Ptr config_tmpl_tags; + String config_tmpl_metric; if (service) { host = service->GetHost(); @@ -168,13 +169,14 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C // Get the tags nested dictionary in the service/host template in the config if (config_tmpl) { config_tmpl_tags = config_tmpl->Get("tags"); + config_tmpl_metric = config_tmpl->Get("metric"); } String metric; std::map tags; // Resolve macros in configuration template and build custom tag list - if (config_tmpl_tags) { + if (config_tmpl_tags || !config_tmpl_metric.IsEmpty()) { // Configure config template macro resolver MacroProcessor::ResolverList resolvers; @@ -183,24 +185,46 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C resolvers.emplace_back("host", host); resolvers.emplace_back("icinga", IcingaApplication::GetInstance()); - ObjectLock olock(config_tmpl_tags); + // Resolve macros for the service and host template config line + if (config_tmpl_tags) { + ObjectLock olock(config_tmpl_tags); + + for (const Dictionary::Pair& pair : config_tmpl_tags) { + + String missing_macro; + Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro); + + if (!missing_macro.IsEmpty()) { + Log(LogDebug, "OpenTsdbWriter") + << "Unable to resolve macro:'" << missing_macro + << "' for this host or service."; + + continue; + } + + String tagname = Convert::ToString(pair.first); + tags[tagname] = EscapeTag(value); + + } + } - for (const Dictionary::Pair& pair : config_tmpl_tags) { + // Resolve macros for the metric config line + if (!config_tmpl_metric.IsEmpty()) { String missing_macro; - Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro); + Value value = MacroProcessor::ResolveMacros(config_tmpl_metric, resolvers, cr, &missing_macro); if (!missing_macro.IsEmpty()) { Log(LogDebug, "OpenTsdbWriter") << "Unable to resolve macro:'" << missing_macro << "' for this host or service."; - continue; } + else { - String tagname = Convert::ToString(pair.first); - tags[tagname] = EscapeTag(value); + config_tmpl_metric = Convert::ToString(value); + } } } @@ -210,13 +234,23 @@ void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const C double ts = cr->GetExecutionEnd(); if (service) { - String serviceName = service->GetShortName(); - String escaped_serviceName = EscapeMetric(serviceName); - metric = "icinga.service." + escaped_serviceName; + if (!config_tmpl_metric.IsEmpty()) { + metric = config_tmpl_metric; + } else { + String serviceName = service->GetShortName(); + String escaped_serviceName = EscapeMetric(serviceName); + metric = "icinga.service." + escaped_serviceName; + } + SendMetric(checkable, metric + ".state", tags, service->GetState(), ts); + } else { - metric = "icinga.host"; + if (!config_tmpl_metric.IsEmpty()) { + metric = config_tmpl_metric; + } else { + metric = "icinga.host"; + } SendMetric(checkable, metric + ".state", tags, host->GetState(), ts); } @@ -280,20 +314,32 @@ void OpenTsdbWriter::SendPerfdata(const Checkable::Ptr& checkable, const String& continue; } } + + String metric_name; + std::map tags_new = tags; - String escaped_key = EscapeMetric(pdv->GetLabel()); - boost::algorithm::replace_all(escaped_key, "::", "."); + // Do not break original functionality where perfdata labels form + // part of the metric name + if (!GetEnableGenericMetrics()) { + String escaped_key = EscapeMetric(pdv->GetLabel()); + boost::algorithm::replace_all(escaped_key, "::", "."); + metric_name = metric + "." + escaped_key; + } else { + String escaped_key = EscapeTag(pdv->GetLabel()); + metric_name = metric; + tags_new["label"] = escaped_key; + } - SendMetric(checkable, metric + "." + escaped_key, tags, pdv->GetValue(), ts); + SendMetric(checkable, metric_name, tags_new, pdv->GetValue(), ts); if (pdv->GetCrit()) - SendMetric(checkable, metric + "." + escaped_key + "_crit", tags, pdv->GetCrit(), ts); + SendMetric(checkable, metric_name + "_crit", tags_new, pdv->GetCrit(), ts); if (pdv->GetWarn()) - SendMetric(checkable, metric + "." + escaped_key + "_warn", tags, pdv->GetWarn(), ts); + SendMetric(checkable, metric_name + "_warn", tags_new, pdv->GetWarn(), ts); if (pdv->GetMin()) - SendMetric(checkable, metric + "." + escaped_key + "_min", tags, pdv->GetMin(), ts); + SendMetric(checkable, metric_name + "_min", tags_new, pdv->GetMin(), ts); if (pdv->GetMax()) - SendMetric(checkable, metric + "." + escaped_key + "_max", tags, pdv->GetMax(), ts); + SendMetric(checkable, metric_name + "_max", tags_new, pdv->GetMax(), ts); } } @@ -360,6 +406,7 @@ String OpenTsdbWriter::EscapeTag(const String& str) boost::replace_all(result, " ", "_"); boost::replace_all(result, "\\", "_"); + boost::replace_all(result, ":", "_"); return result; } @@ -427,6 +474,10 @@ void OpenTsdbWriter::ValidateHostTemplate(const Lazy& lvalue, c { ObjectImpl::ValidateHostTemplate(lvalue, utils); + String metric = lvalue()->Get("metric"); + if (!MacroProcessor::ValidateMacroString(metric)) + BOOST_THROW_EXCEPTION(ValidationError(this, { "host_template", "metric" }, "Closing $ not found in macro format string '" + metric + "'.")); + Dictionary::Ptr tags = lvalue()->Get("tags"); if (tags) { ObjectLock olock(tags); @@ -448,6 +499,10 @@ void OpenTsdbWriter::ValidateServiceTemplate(const Lazy& lvalue { ObjectImpl::ValidateServiceTemplate(lvalue, utils); + String metric = lvalue()->Get("metric"); + if (!MacroProcessor::ValidateMacroString(metric)) + BOOST_THROW_EXCEPTION(ValidationError(this, { "service_template", "metric" }, "Closing $ not found in macro format string '" + metric + "'.")); + Dictionary::Ptr tags = lvalue()->Get("tags"); if (tags) { ObjectLock olock(tags); diff --git a/lib/perfdata/opentsdbwriter.ti b/lib/perfdata/opentsdbwriter.ti index 7a0219f01..626350a80 100644 --- a/lib/perfdata/opentsdbwriter.ti +++ b/lib/perfdata/opentsdbwriter.ti @@ -27,6 +27,9 @@ class OpenTsdbWriter : ConfigObject [config] Dictionary::Ptr service_template { default {{{ return new Dictionary(); }}} }; + [config] bool enable_generic_metrics { + default {{{ return false; }}} + }; [no_user_modify] bool connected; [no_user_modify] bool should_connect { @@ -36,11 +39,13 @@ class OpenTsdbWriter : ConfigObject validator OpenTsdbWriter { Dictionary host_template { + String metric; Dictionary "tags" { String "*"; }; }; Dictionary service_template { + String metric; Dictionary "tags" { String "*"; };