diff --git a/doc/09-object-types.md b/doc/09-object-types.md index c2d5e5eff..67735e986 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1181,7 +1181,7 @@ Configuration Attributes: ### ElasticsearchWriter -Writes check result metrics and performance data to an Elasticsearch instance. +Writes check result metrics and performance data to an Elasticsearch or OpenSearch instance. This configuration object is available as [elasticsearch feature](14-features.md#elasticsearch-writer). Example: @@ -1194,6 +1194,10 @@ object ElasticsearchWriter "elasticsearch" { enable_send_perfdata = true + host_tags_template = { + os_name = "$host.vars.os$" + } + flush_threshold = 1024 flush_interval = 10 } @@ -1215,6 +1219,8 @@ Configuration Attributes: password | String | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy. enable\_tls | Boolean | **Optional.** Whether to use a TLS stream. Defaults to `false`. Requires an HTTP proxy. insecure\_noverify | Boolean | **Optional.** Disable TLS peer verification. + host\_tags\_template | Dictionary | **Optional.** Allows to apply additional tags to the Elasticsearch host entries. + service\_tags\_template | Dictionary | **Optional.** Allows to apply additional tags to the Elasticsearch service entries. ca\_path | String | **Optional.** Path to CA certificate to validate the remote host. Requires `enable_tls` set to `true`. cert\_path | String | **Optional.** Path to host certificate to present to the remote host for mutual verification. Requires `enable_tls` set to `true`. key\_path | String | **Optional.** Path to host key to accompany the cert\_path. Requires `enable_tls` set to `true`. diff --git a/doc/14-features.md b/doc/14-features.md index 6ad4d1095..31c696619 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -335,16 +335,14 @@ More integrations: #### Elasticsearch Writer This feature forwards check results, state changes and notification events -to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) installation over its HTTP API. +to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) or an [OpenSearch](https://opensearch.org/) installation over its HTTP API. The check results include parsed performance data metrics if enabled. > **Note** > -> Elasticsearch 5.x or 6.x are required. This feature has been successfully tested with -> Elasticsearch 5.6.7 and 6.3.1. - - +> Elasticsearch 7.x, 8.x or Opensearch 2.12.x are required. This feature has been successfully tested with +> Elasticsearch 7.17.10, 8.8.1 and OpenSearch 2.13.0. Enable the feature and restart Icinga 2. @@ -398,6 +396,28 @@ check_result.perfdata..warn check_result.perfdata..crit ``` +Additionaly it is possible to configure custom tags that are applied to the metrics via `host_tags_template` or `service_tags_template`. +Depending on whether the write event was triggered on a service or host object, additional tags are added to the ElasticSearch entries. + +A host metrics entry configured with the following `host_tags_template`: + +``` +host_tags_template = { + + os_name = "$host.vars.os$" + custom_label = "A Custom Label" + list = [ "$host.groups$", "$host.vars.foo$" ] +} +``` + +Will in addition to the above mentioned lines also contain: + +``` +os_name = "Linux" +custom_label = "A Custom Label" +list = [ "group-A;linux-servers", "bar" ] +``` + #### Elasticsearch in Cluster HA Zones The Elasticsearch feature supports [high availability](06-distributed-monitoring.md#distributed-monitoring-high-availability-features) diff --git a/lib/perfdata/elasticsearchwriter.cpp b/lib/perfdata/elasticsearchwriter.cpp index 9fb2aa90f..eaa19da59 100644 --- a/lib/perfdata/elasticsearchwriter.cpp +++ b/lib/perfdata/elasticsearchwriter.cpp @@ -5,6 +5,7 @@ #include "remote/url.hpp" #include "icinga/compatutility.hpp" #include "icinga/service.hpp" +#include "icinga/macroprocessor.hpp" #include "icinga/checkcommand.hpp" #include "base/application.hpp" #include "base/defer.hpp" @@ -131,6 +132,33 @@ void ElasticsearchWriter::Pause() ObjectImpl::Pause(); } +void ElasticsearchWriter::AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr) +{ + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(checkable); + + Dictionary::Ptr tmpl = service ? GetServiceTagsTemplate() : GetHostTagsTemplate(); + + if (tmpl) { + MacroProcessor::ResolverList resolvers; + resolvers.emplace_back("host", host); + if (service) { + resolvers.emplace_back("service", service); + } + + ObjectLock olock(tmpl); + for (const Dictionary::Pair& pair : tmpl) { + String missingMacro; + Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missingMacro); + + if (missingMacro.IsEmpty()) { + fields->Set(pair.first, value); + } + } + } +} + void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr) { String prefix = "check_result."; @@ -257,6 +285,8 @@ void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& check ts = cr->GetExecutionEnd(); } + AddTemplateTags(fields, checkable, cr); + Enqueue(checkable, "checkresult", fields, ts); } @@ -307,6 +337,8 @@ void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& check ts = cr->GetExecutionEnd(); } + AddTemplateTags(fields, checkable, cr); + Enqueue(checkable, "statechange", fields, ts); } @@ -377,6 +409,8 @@ void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notifi ts = cr->GetExecutionEnd(); } + AddTemplateTags(fields, checkable, cr); + Enqueue(checkable, "notification", fields, ts); } @@ -683,3 +717,49 @@ String ElasticsearchWriter::FormatTimestamp(double ts) return Utility::FormatDateTime("%Y-%m-%dT%H:%M:%S", ts) + "." + Convert::ToString(milliSeconds) + Utility::FormatDateTime("%z", ts); } + +void ElasticsearchWriter::ValidateHostTagsTemplate(const Lazy& lvalue, const ValidationUtils& utils) +{ + ObjectImpl::ValidateHostTagsTemplate(lvalue, utils); + + Dictionary::Ptr tags = lvalue(); + if (tags) { + ObjectLock olock(tags); + for (const Dictionary::Pair& pair : tags) { + if (pair.second.IsObjectType()) { + Array::Ptr arrObject = pair.second; + ObjectLock arrLock(arrObject); + for (const Value& arrValue : arrObject) { + if (!MacroProcessor::ValidateMacroString(arrValue)) { + BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'.")); + } + } + } else if (!MacroProcessor::ValidateMacroString(pair.second)) { + BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'.")); + } + } + } +} + +void ElasticsearchWriter::ValidateServiceTagsTemplate(const Lazy& lvalue, const ValidationUtils& utils) +{ + ObjectImpl::ValidateServiceTagsTemplate(lvalue, utils); + + Dictionary::Ptr tags = lvalue(); + if (tags) { + ObjectLock olock(tags); + for (const Dictionary::Pair& pair : tags) { + if (pair.second.IsObjectType()) { + Array::Ptr arrObject = pair.second; + ObjectLock arrLock(arrObject); + for (const Value& arrValue : arrObject) { + if (!MacroProcessor::ValidateMacroString(arrValue)) { + BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'.")); + } + } + } else if (!MacroProcessor::ValidateMacroString(pair.second)) { + BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'.")); + } + } + } +} diff --git a/lib/perfdata/elasticsearchwriter.hpp b/lib/perfdata/elasticsearchwriter.hpp index a988094d8..9ecd0fd76 100644 --- a/lib/perfdata/elasticsearchwriter.hpp +++ b/lib/perfdata/elasticsearchwriter.hpp @@ -23,6 +23,9 @@ public: static String FormatTimestamp(double ts); + void ValidateHostTagsTemplate(const Lazy &lvalue, const ValidationUtils &utils) override; + void ValidateServiceTagsTemplate(const Lazy &lvalue, const ValidationUtils &utils) override; + protected: void OnConfigLoaded() override; void Resume() override; @@ -37,6 +40,7 @@ private: std::mutex m_DataBufferMutex; void AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr); + void AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr); void StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type); void StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type); diff --git a/lib/perfdata/elasticsearchwriter.ti b/lib/perfdata/elasticsearchwriter.ti index e3b8e27f5..ed9b24fb5 100644 --- a/lib/perfdata/elasticsearchwriter.ti +++ b/lib/perfdata/elasticsearchwriter.ti @@ -29,6 +29,9 @@ class ElasticsearchWriter : ConfigObject [config] bool enable_tls { default {{{ return false; }}} }; + + [config] Dictionary::Ptr host_tags_template; + [config] Dictionary::Ptr service_tags_template; [config] bool insecure_noverify { default {{{ return false; }}} }; @@ -47,4 +50,19 @@ class ElasticsearchWriter : ConfigObject }; }; +validator ElasticsearchWriter { + Dictionary host_tags_template { + String "*"; + Array "*" { + String "*"; + }; + }; + Dictionary service_tags_template { + String "*"; + Array "*" { + String "*"; + }; + }; +}; + }