From 9d4b0f12687acc4fa83ef3f9e7b36db1dc32f2c0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 15 Apr 2021 14:08:50 +0200 Subject: [PATCH] Introduce Influxdb2Writer refs #8711 --- doc/06-distributed-monitoring.md | 2 +- doc/09-object-types.md | 63 +++++++++++++++++++ doc/13-addons.md | 7 +++ doc/14-features.md | 20 ++++-- etc/icinga2/features-available/influxdb2.conf | 27 ++++++++ lib/perfdata/CMakeLists.txt | 7 +++ lib/perfdata/influxdb2writer.cpp | 44 +++++++++++++ lib/perfdata/influxdb2writer.hpp | 33 ++++++++++ lib/perfdata/influxdb2writer.ti | 19 ++++++ tools/syntax/vim/syntax/icinga2.vim | 2 +- 10 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 etc/icinga2/features-available/influxdb2.conf create mode 100644 lib/perfdata/influxdb2writer.cpp create mode 100644 lib/perfdata/influxdb2writer.hpp create mode 100644 lib/perfdata/influxdb2writer.ti diff --git a/doc/06-distributed-monitoring.md b/doc/06-distributed-monitoring.md index 1531451ba..ca47bccb5 100644 --- a/doc/06-distributed-monitoring.md +++ b/doc/06-distributed-monitoring.md @@ -3001,7 +3001,7 @@ By default, the following features provide advanced HA functionality: * [Elasticsearch](09-object-types.md#objecttype-elasticsearchwriter) * [Gelf](09-object-types.md#objecttype-gelfwriter) * [Graphite](09-object-types.md#objecttype-graphitewriter) -* [InfluxDB](09-object-types.md#objecttype-influxdbwriter) +* [InfluxDB](09-object-types.md#objecttype-influxdb2writer) (v1 and v2) * [OpenTsdb](09-object-types.md#objecttype-opentsdbwriter) * [Perfdata](09-object-types.md#objecttype-perfdatawriter) (for PNP) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index b87eff813..19e796aff 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1606,6 +1606,7 @@ Runtime Attributes: Writes check result metrics and performance data to a defined InfluxDB v1 host. This configuration object is available as [influxdb feature](14-features.md#influxdb-writer). +For InfluxDB v2 support see the [Influxdb2Writer](#objecttype-influxdb2writer) below. Example: @@ -1669,6 +1670,68 @@ or similar. +### Influxdb2Writer + +Writes check result metrics and performance data to a defined InfluxDB v2 host. +This configuration object is available as [influxdb feature](14-features.md#influxdb-writer). +For InfluxDB v1 support see the [InfluxdbWriter](#objecttype-influxdbwriter) above. + +Example: + +``` +object Influxdb2Writer "influxdb2" { + host = "127.0.0.1" + port = 8086 + organization = "monitoring" + bucket = "icinga2" + auth_token = "ABCDEvwxyz0189-_" + + flush_threshold = 1024 + flush_interval = 10s + + host_template = { + measurement = "$host.check_command$" + tags = { + hostname = "$host.name$" + } + } + service_template = { + measurement = "$service.check_command$" + tags = { + hostname = "$host.name$" + service = "$service.name$" + } + } +} +``` + +Configuration Attributes: + + Name | Type | Description + --------------------------|-----------------------|---------------------------------- + host | String | **Required.** InfluxDB host address. Defaults to `127.0.0.1`. + port | Number | **Required.** InfluxDB HTTP port. Defaults to `8086`. + organization | String | **Required.** InfluxDB organization name. + bucket | String | **Required.** InfluxDB bucket name. + auth\_token | String | **Required.** InfluxDB authentication token. + ssl\_enable | Boolean | **Optional.** Whether to use a TLS stream. Defaults to `false`. + ssl\_ca\_cert | String | **Optional.** Path to CA certificate to validate the remote host. + ssl\_cert | String | **Optional.** Path to host certificate to present to the remote host for mutual verification. + ssl\_key | String | **Optional.** Path to host key to accompany the ssl\_cert. + host\_template | Dictionary | **Required.** Host template to define the InfluxDB line protocol. + service\_template | Dictionary | **Required.** Service template to define the influxDB line protocol. + enable\_send\_thresholds | Boolean | **Optional.** Whether to send warn, crit, min & max tagged data. + enable\_send\_metadata | Boolean | **Optional.** Whether to send check metadata e.g. states, execution time, latency etc. + flush\_interval | Duration | **Optional.** How long to buffer data points before transferring to InfluxDB. Defaults to `10s`. + flush\_threshold | Number | **Optional.** How many data points to buffer before forcing a transfer to InfluxDB. Defaults to `1024`. + enable\_ha | Boolean | **Optional.** Enable the high availability functionality. Only valid in a [cluster setup](06-distributed-monitoring.md#distributed-monitoring-high-availability-features). Defaults to `false`. + +Note: If `flush_threshold` is set too low, this will always force the feature to flush all data +to InfluxDB. Experiment with the setting, if you are processing more than 1024 metrics per second +or similar. + + + ### LiveStatusListener Livestatus API interface available as TCP or UNIX socket. Historical table queries diff --git a/doc/13-addons.md b/doc/13-addons.md index fd65b4b02..e212c3734 100644 --- a/doc/13-addons.md +++ b/doc/13-addons.md @@ -63,6 +63,13 @@ for sending real-time metrics from Icinga 2 to InfluxDB v1. icinga2 feature enable influxdb ``` +Use the [Influxdb2Writer](14-features.md#influxdb-writer) feature +for sending real-time metrics from Icinga 2 to InfluxDB v2. + +```bash +icinga2 feature enable influxdb2 +``` + A popular frontend for InfluxDB is for example [Grafana](https://grafana.org). Integration in Icinga Web 2 is possible by installing the community [Grafana module](https://github.com/Mikesch-mp/icingaweb2-module-grafana). diff --git a/doc/14-features.md b/doc/14-features.md index 10656331a..5e5eea0a7 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -369,7 +369,7 @@ where Carbon Cache/Relay is running as receiver. ### InfluxDB Writer Once there are new metrics available, Icinga 2 will directly write them to the -defined InfluxDB v1 HTTP API. +defined InfluxDB v1/v2 HTTP API. You can enable the feature using @@ -377,10 +377,17 @@ You can enable the feature using icinga2 feature enable influxdb ``` -By default the [InfluxdbWriter](09-object-types.md#objecttype-influxdbwriter) feature -expects the InfluxDB daemon to listen at `127.0.0.1` on port `8086`. +or -Measurement names and tags are fully configurable by the end user. The InfluxdbWriter +```bash +icinga2 feature enable influxdb2 +``` + +By default the +[InfluxdbWriter](09-object-types.md#objecttype-influxdbwriter)/[Influxdb2Writer](09-object-types.md#objecttype-influxdb2writer) +features expect the InfluxDB daemon to listen at `127.0.0.1` on port `8086`. + +Measurement names and tags are fully configurable by the end user. The Influxdb(2)Writer object will automatically add a `metric` tag to each data point. This correlates to the perfdata label. Fields (value, warn, crit, min, max, unit) are created from data if available and the configuration allows it. If a value associated with a tag is not able to be @@ -391,12 +398,13 @@ escape characters when followed by a space or comma, but cannot be escaped thems As a result all trailling slashes in these fields are replaced with an underscore. This predominantly affects Windows paths e.g. `C:\` becomes `C:_`. -The database is assumed to exist so this object will make no attempt to create it currently. +The database/bucket is assumed to exist so this object will make no attempt to create it currently. If [SELinux](22-selinux.md#selinux) is enabled, it will not allow access for Icinga 2 to InfluxDB until the [boolean](22-selinux.md#selinux-policy-booleans) `icinga2_can_connect_all` is set to true as InfluxDB is not providing its own policy. -More configuration details can be found [here](09-object-types.md#objecttype-influxdbwriter). +More configuration details can be found [here for v1](09-object-types.md#objecttype-influxdbwriter) +and [here for v2](09-object-types.md#objecttype-influxdb2writer). #### Instance Tagging diff --git a/etc/icinga2/features-available/influxdb2.conf b/etc/icinga2/features-available/influxdb2.conf new file mode 100644 index 000000000..53f7a217b --- /dev/null +++ b/etc/icinga2/features-available/influxdb2.conf @@ -0,0 +1,27 @@ +/** + * The Influxdb2Writer type writes check result metrics and + * performance data to an InfluxDB v2 HTTP API + */ + +object Influxdb2Writer "influxdb2" { + //host = "127.0.0.1" + //port = 8086 + //organization = "monitoring" + //bucket = "icinga2" + //auth_token = "ABCDEvwxyz0189-_" + //flush_threshold = 1024 + //flush_interval = 10s + //host_template = { + // measurement = "$host.check_command$" + // tags = { + // hostname = "$host.name$" + // } + //} + //service_template = { + // measurement = "$service.check_command$" + // tags = { + // hostname = "$host.name$" + // service = "$service.name$" + // } + //} +} diff --git a/lib/perfdata/CMakeLists.txt b/lib/perfdata/CMakeLists.txt index 0e54ec832..168938c42 100644 --- a/lib/perfdata/CMakeLists.txt +++ b/lib/perfdata/CMakeLists.txt @@ -4,6 +4,7 @@ mkclass_target(gelfwriter.ti gelfwriter-ti.cpp gelfwriter-ti.hpp) mkclass_target(graphitewriter.ti graphitewriter-ti.cpp graphitewriter-ti.hpp) mkclass_target(influxdbcommonwriter.ti influxdbcommonwriter-ti.cpp influxdbcommonwriter-ti.hpp) mkclass_target(influxdbwriter.ti influxdbwriter-ti.cpp influxdbwriter-ti.hpp) +mkclass_target(influxdb2writer.ti influxdb2writer-ti.cpp influxdb2writer-ti.hpp) mkclass_target(elasticsearchwriter.ti elasticsearchwriter-ti.cpp elasticsearchwriter-ti.hpp) mkclass_target(opentsdbwriter.ti opentsdbwriter-ti.cpp opentsdbwriter-ti.hpp) mkclass_target(perfdatawriter.ti perfdatawriter-ti.cpp perfdatawriter-ti.hpp) @@ -14,6 +15,7 @@ set(perfdata_SOURCES graphitewriter.cpp graphitewriter.hpp graphitewriter-ti.hpp influxdbcommonwriter.cpp influxdbcommonwriter.hpp influxdbcommonwriter-ti.hpp influxdbwriter.cpp influxdbwriter.hpp influxdbwriter-ti.hpp + influxdb2writer.cpp influxdb2writer.hpp influxdb2writer-ti.hpp opentsdbwriter.cpp opentsdbwriter.hpp opentsdbwriter-ti.hpp perfdatawriter.cpp perfdatawriter.hpp perfdatawriter-ti.hpp ) @@ -46,6 +48,11 @@ install_if_not_exists( ${ICINGA2_CONFIGDIR}/features-available ) +install_if_not_exists( + ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/influxdb2.conf + ${ICINGA2_CONFIGDIR}/features-available +) + install_if_not_exists( ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/elasticsearch.conf ${ICINGA2_CONFIGDIR}/features-available diff --git a/lib/perfdata/influxdb2writer.cpp b/lib/perfdata/influxdb2writer.cpp new file mode 100644 index 000000000..57fc94e23 --- /dev/null +++ b/lib/perfdata/influxdb2writer.cpp @@ -0,0 +1,44 @@ +/* Icinga 2 | (c) 2021 Icinga GmbH | GPLv2+ */ + +#include "perfdata/influxdb2writer.hpp" +#include "perfdata/influxdb2writer-ti.cpp" +#include "remote/url.hpp" +#include "base/configtype.hpp" +#include "base/perfdatavalue.hpp" +#include "base/statsfunction.hpp" +#include +#include +#include + +using namespace icinga; + +REGISTER_TYPE(Influxdb2Writer); + +REGISTER_STATSFUNCTION(Influxdb2Writer, &Influxdb2Writer::StatsFunc); + +void Influxdb2Writer::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata) +{ + InfluxdbCommonWriter::StatsFunc(status, perfdata); +} + +boost::beast::http::request Influxdb2Writer::AssembleRequest(String body) +{ + auto request (AssembleBaseRequest(std::move(body))); + + request.set(boost::beast::http::field::authorization, "Token " + GetAuthToken()); + + return std::move(request); +} + +Url::Ptr Influxdb2Writer::AssembleUrl() +{ + auto url (AssembleBaseUrl()); + + std::vector path ({"api", "v2", "write"}); + url->SetPath(path); + + url->AddQueryElement("org", GetOrganization()); + url->AddQueryElement("bucket", GetBucket()); + + return std::move(url); +} diff --git a/lib/perfdata/influxdb2writer.hpp b/lib/perfdata/influxdb2writer.hpp new file mode 100644 index 000000000..3b20f8b9d --- /dev/null +++ b/lib/perfdata/influxdb2writer.hpp @@ -0,0 +1,33 @@ +/* Icinga 2 | (c) 2021 Icinga GmbH | GPLv2+ */ + +#ifndef INFLUXDB2WRITER_H +#define INFLUXDB2WRITER_H + +#include "perfdata/influxdb2writer-ti.hpp" +#include +#include + +namespace icinga +{ + +/** + * An Icinga InfluxDB v2 writer. + * + * @ingroup perfdata + */ +class Influxdb2Writer final : public ObjectImpl +{ +public: + DECLARE_OBJECT(Influxdb2Writer); + DECLARE_OBJECTNAME(Influxdb2Writer); + + static void StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& perfdata); + +protected: + boost::beast::http::request AssembleRequest(String body) override; + Url::Ptr AssembleUrl() override; +}; + +} + +#endif /* INFLUXDB2WRITER_H */ diff --git a/lib/perfdata/influxdb2writer.ti b/lib/perfdata/influxdb2writer.ti new file mode 100644 index 000000000..f80618755 --- /dev/null +++ b/lib/perfdata/influxdb2writer.ti @@ -0,0 +1,19 @@ +/* Icinga 2 | (c) 2021 Icinga GmbH | GPLv2+ */ + +#include "perfdata/influxdbcommonwriter.hpp" + +library perfdata; + +namespace icinga +{ + +class Influxdb2Writer : InfluxdbCommonWriter +{ + activation_priority 100; + + [config, required] String organization; + [config, required] String bucket; + [config, required, no_user_view] String auth_token; +}; + +} diff --git a/tools/syntax/vim/syntax/icinga2.vim b/tools/syntax/vim/syntax/icinga2.vim index 59ff202f1..3cbb8e632 100644 --- a/tools/syntax/vim/syntax/icinga2.vim +++ b/tools/syntax/vim/syntax/icinga2.vim @@ -57,7 +57,7 @@ syn keyword icinga2ObjType Comment Dependency Downtime ElasticsearchWriter syn keyword icinga2ObjType Endpoint EventCommand ExternalCommandListener syn keyword icinga2ObjType FileLogger GelfWriter GraphiteWriter Host HostGroup syn keyword icinga2ObjType IcingaApplication IdoMysqlConnection IdoPgsqlConnection -syn keyword icinga2ObjType InfluxdbWriter LivestatusListener Notification NotificationCommand +syn keyword icinga2ObjType InfluxdbWriter Influxdb2Writer LivestatusListener Notification NotificationCommand syn keyword icinga2ObjType NotificationComponent OpenTsdbWriter PerfdataWriter syn keyword icinga2ObjType ScheduledDowntime Service ServiceGroup SyslogLogger syn keyword icinga2ObjType TimePeriod User UserGroup WindowsEventLogLogger Zone