Merge pull request #10074 from open-i-gmbh/feature/tags-for-elasticsearchwriter-6837

Feature/tags for elasticsearchwriter
This commit is contained in:
Julian Brost 2025-04-10 10:11:15 +02:00 committed by GitHub
commit 8d607d2ef7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 134 additions and 6 deletions

View File

@ -1181,7 +1181,7 @@ Configuration Attributes:
### ElasticsearchWriter <a id="objecttype-elasticsearchwriter"></a>
Writes check result metrics and performance data to an Elasticsearch instance.
Writes check result metrics and performance data to an Elasticsearch or OpenSearch instance.
This configuration object is available as [elasticsearch feature](14-features.md#elasticsearch-writer).
Example:
@ -1194,6 +1194,10 @@ object ElasticsearchWriter "elasticsearch" {
enable_send_perfdata = true
host_tags_template = {
os_name = "$host.vars.os$"
}
flush_threshold = 1024
flush_interval = 10
}
@ -1215,6 +1219,8 @@ Configuration Attributes:
password | String | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy.
enable\_tls | Boolean | **Optional.** Whether to use a TLS stream. Defaults to `false`. Requires an HTTP proxy.
insecure\_noverify | Boolean | **Optional.** Disable TLS peer verification.
host\_tags\_template | Dictionary | **Optional.** Allows to apply additional tags to the Elasticsearch host entries.
service\_tags\_template | Dictionary | **Optional.** Allows to apply additional tags to the Elasticsearch service entries.
ca\_path | String | **Optional.** Path to CA certificate to validate the remote host. Requires `enable_tls` set to `true`.
cert\_path | String | **Optional.** Path to host certificate to present to the remote host for mutual verification. Requires `enable_tls` set to `true`.
key\_path | String | **Optional.** Path to host key to accompany the cert\_path. Requires `enable_tls` set to `true`.

View File

@ -335,16 +335,14 @@ More integrations:
#### Elasticsearch Writer <a id="elasticsearch-writer"></a>
This feature forwards check results, state changes and notification events
to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) installation over its HTTP API.
to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) or an [OpenSearch](https://opensearch.org/) installation over its HTTP API.
The check results include parsed performance data metrics if enabled.
> **Note**
>
> Elasticsearch 5.x or 6.x are required. This feature has been successfully tested with
> Elasticsearch 5.6.7 and 6.3.1.
> Elasticsearch 7.x, 8.x or Opensearch 2.12.x are required. This feature has been successfully tested with
> Elasticsearch 7.17.10, 8.8.1 and OpenSearch 2.13.0.
Enable the feature and restart Icinga 2.
@ -398,6 +396,28 @@ check_result.perfdata.<perfdata-label>.warn
check_result.perfdata.<perfdata-label>.crit
```
Additionaly it is possible to configure custom tags that are applied to the metrics via `host_tags_template` or `service_tags_template`.
Depending on whether the write event was triggered on a service or host object, additional tags are added to the ElasticSearch entries.
A host metrics entry configured with the following `host_tags_template`:
```
host_tags_template = {
os_name = "$host.vars.os$"
custom_label = "A Custom Label"
list = [ "$host.groups$", "$host.vars.foo$" ]
}
```
Will in addition to the above mentioned lines also contain:
```
os_name = "Linux"
custom_label = "A Custom Label"
list = [ "group-A;linux-servers", "bar" ]
```
#### Elasticsearch in Cluster HA Zones <a id="elasticsearch-writer-cluster-ha"></a>
The Elasticsearch feature supports [high availability](06-distributed-monitoring.md#distributed-monitoring-high-availability-features)

View File

@ -5,6 +5,7 @@
#include "remote/url.hpp"
#include "icinga/compatutility.hpp"
#include "icinga/service.hpp"
#include "icinga/macroprocessor.hpp"
#include "icinga/checkcommand.hpp"
#include "base/application.hpp"
#include "base/defer.hpp"
@ -131,6 +132,33 @@ void ElasticsearchWriter::Pause()
ObjectImpl<ElasticsearchWriter>::Pause();
}
void ElasticsearchWriter::AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr tmpl = service ? GetServiceTagsTemplate() : GetHostTagsTemplate();
if (tmpl) {
MacroProcessor::ResolverList resolvers;
resolvers.emplace_back("host", host);
if (service) {
resolvers.emplace_back("service", service);
}
ObjectLock olock(tmpl);
for (const Dictionary::Pair& pair : tmpl) {
String missingMacro;
Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missingMacro);
if (missingMacro.IsEmpty()) {
fields->Set(pair.first, value);
}
}
}
}
void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
String prefix = "check_result.";
@ -257,6 +285,8 @@ void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& check
ts = cr->GetExecutionEnd();
}
AddTemplateTags(fields, checkable, cr);
Enqueue(checkable, "checkresult", fields, ts);
}
@ -307,6 +337,8 @@ void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& check
ts = cr->GetExecutionEnd();
}
AddTemplateTags(fields, checkable, cr);
Enqueue(checkable, "statechange", fields, ts);
}
@ -377,6 +409,8 @@ void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notifi
ts = cr->GetExecutionEnd();
}
AddTemplateTags(fields, checkable, cr);
Enqueue(checkable, "notification", fields, ts);
}
@ -683,3 +717,49 @@ String ElasticsearchWriter::FormatTimestamp(double ts)
return Utility::FormatDateTime("%Y-%m-%dT%H:%M:%S", ts) + "." + Convert::ToString(milliSeconds) + Utility::FormatDateTime("%z", ts);
}
void ElasticsearchWriter::ValidateHostTagsTemplate(const Lazy<Dictionary::Ptr>& lvalue, const ValidationUtils& utils)
{
ObjectImpl<ElasticsearchWriter>::ValidateHostTagsTemplate(lvalue, utils);
Dictionary::Ptr tags = lvalue();
if (tags) {
ObjectLock olock(tags);
for (const Dictionary::Pair& pair : tags) {
if (pair.second.IsObjectType<Array>()) {
Array::Ptr arrObject = pair.second;
ObjectLock arrLock(arrObject);
for (const Value& arrValue : arrObject) {
if (!MacroProcessor::ValidateMacroString(arrValue)) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'."));
}
}
} else if (!MacroProcessor::ValidateMacroString(pair.second)) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'."));
}
}
}
}
void ElasticsearchWriter::ValidateServiceTagsTemplate(const Lazy<Dictionary::Ptr>& lvalue, const ValidationUtils& utils)
{
ObjectImpl<ElasticsearchWriter>::ValidateServiceTagsTemplate(lvalue, utils);
Dictionary::Ptr tags = lvalue();
if (tags) {
ObjectLock olock(tags);
for (const Dictionary::Pair& pair : tags) {
if (pair.second.IsObjectType<Array>()) {
Array::Ptr arrObject = pair.second;
ObjectLock arrLock(arrObject);
for (const Value& arrValue : arrObject) {
if (!MacroProcessor::ValidateMacroString(arrValue)) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'."));
}
}
} else if (!MacroProcessor::ValidateMacroString(pair.second)) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'."));
}
}
}
}

View File

@ -23,6 +23,9 @@ public:
static String FormatTimestamp(double ts);
void ValidateHostTagsTemplate(const Lazy<Dictionary::Ptr> &lvalue, const ValidationUtils &utils) override;
void ValidateServiceTagsTemplate(const Lazy<Dictionary::Ptr> &lvalue, const ValidationUtils &utils) override;
protected:
void OnConfigLoaded() override;
void Resume() override;
@ -37,6 +40,7 @@ private:
std::mutex m_DataBufferMutex;
void AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
void AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
void StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type);
void StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type);

View File

@ -29,6 +29,9 @@ class ElasticsearchWriter : ConfigObject
[config] bool enable_tls {
default {{{ return false; }}}
};
[config] Dictionary::Ptr host_tags_template;
[config] Dictionary::Ptr service_tags_template;
[config] bool insecure_noverify {
default {{{ return false; }}}
};
@ -47,4 +50,19 @@ class ElasticsearchWriter : ConfigObject
};
};
validator ElasticsearchWriter {
Dictionary host_tags_template {
String "*";
Array "*" {
String "*";
};
};
Dictionary service_tags_template {
String "*";
Array "*" {
String "*";
};
};
};
}