From d74b15d6408cb4644e7d9a27ae76d67e7a3dd57f Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 11 Sep 2017 16:51:13 +0200 Subject: [PATCH 1/3] Move Base64 class into libbase refs #5538 --- lib/base/CMakeLists.txt | 2 +- lib/{remote => base}/base64.cpp | 2 +- lib/{remote => base}/base64.hpp | 2 +- lib/remote/CMakeLists.txt | 2 +- lib/remote/apiclient.cpp | 2 +- lib/remote/httpclientconnection.cpp | 2 +- lib/remote/httpserverconnection.cpp | 2 +- test/CMakeLists.txt | 6 +++--- test/{remote-base64.cpp => base-base64.cpp} | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) rename lib/{remote => base}/base64.cpp (98%) rename lib/{remote => base}/base64.hpp (98%) rename test/{remote-base64.cpp => base-base64.cpp} (97%) diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index 858943f81..049226e9c 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -27,7 +27,7 @@ mkclass_target(sysloglogger.ti sysloglogger.tcpp sysloglogger.thpp) set(base_SOURCES application.cpp application.thpp application-version.cpp array.cpp - array-script.cpp boolean.cpp boolean-script.cpp console.cpp context.cpp + array-script.cpp boolean.cpp boolean-script.cpp base64.cpp console.cpp context.cpp convert.cpp datetime.cpp datetime.thpp datetime-script.cpp debuginfo.cpp dictionary.cpp dictionary-script.cpp configobject.cpp configobject.thpp configobject-script.cpp configtype.cpp configwriter.cpp dependencygraph.cpp exception.cpp fifo.cpp filelogger.cpp filelogger.thpp initialize.cpp json.cpp diff --git a/lib/remote/base64.cpp b/lib/base/base64.cpp similarity index 98% rename from lib/remote/base64.cpp rename to lib/base/base64.cpp index 9c805e658..bbbc9b09e 100644 --- a/lib/remote/base64.cpp +++ b/lib/base/base64.cpp @@ -17,7 +17,7 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "remote/base64.hpp" +#include "base/base64.hpp" #include #include #include diff --git a/lib/remote/base64.hpp b/lib/base/base64.hpp similarity index 98% rename from lib/remote/base64.hpp rename to lib/base/base64.hpp index f9f878b75..c0a127f89 100644 --- a/lib/remote/base64.hpp +++ b/lib/base/base64.hpp @@ -31,7 +31,7 @@ namespace icinga * * @ingroup remote */ -struct I2_REMOTE_API Base64 +struct I2_BASE_API Base64 { static String Decode(const String& data); static String Encode(const String& data); diff --git a/lib/remote/CMakeLists.txt b/lib/remote/CMakeLists.txt index eb9867368..cf27ac463 100644 --- a/lib/remote/CMakeLists.txt +++ b/lib/remote/CMakeLists.txt @@ -23,7 +23,7 @@ mkclass_target(zone.ti zone.tcpp zone.thpp) set(remote_SOURCES actionshandler.cpp apiaction.cpp apiclient.cpp apifunction.cpp apilistener.cpp apilistener.thpp apilistener-configsync.cpp - apilistener-filesync.cpp apiuser.cpp apiuser.thpp authority.cpp base64.cpp + apilistener-filesync.cpp apiuser.cpp apiuser.thpp authority.cpp consolehandler.cpp configfileshandler.cpp configpackageshandler.cpp configpackageutility.cpp configobjectutility.cpp configstageshandler.cpp createobjecthandler.cpp deleteobjecthandler.cpp endpoint.cpp endpoint.thpp eventshandler.cpp eventqueue.cpp filterutility.cpp diff --git a/lib/remote/apiclient.cpp b/lib/remote/apiclient.cpp index 9625298e5..8aa676a47 100644 --- a/lib/remote/apiclient.cpp +++ b/lib/remote/apiclient.cpp @@ -18,7 +18,7 @@ ******************************************************************************/ #include "remote/apiclient.hpp" -#include "remote/base64.hpp" +#include "base/base64.hpp" #include "base/json.hpp" #include "base/logger.hpp" #include "base/exception.hpp" diff --git a/lib/remote/httpclientconnection.cpp b/lib/remote/httpclientconnection.cpp index 03f52d9b9..ffd241caa 100644 --- a/lib/remote/httpclientconnection.cpp +++ b/lib/remote/httpclientconnection.cpp @@ -18,9 +18,9 @@ ******************************************************************************/ #include "remote/httpclientconnection.hpp" -#include "remote/base64.hpp" #include "base/configtype.hpp" #include "base/objectlock.hpp" +#include "base/base64.hpp" #include "base/utility.hpp" #include "base/logger.hpp" #include "base/exception.hpp" diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index f68695a75..aac35aaa1 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -23,7 +23,7 @@ #include "remote/apilistener.hpp" #include "remote/apifunction.hpp" #include "remote/jsonrpc.hpp" -#include "remote/base64.hpp" +#include "base/base64.hpp" #include "base/configtype.hpp" #include "base/objectlock.hpp" #include "base/utility.hpp" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5c79ce3ef..85b8b59cb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,13 +20,13 @@ set(Boost_USE_STATIC_LIBS OFF) include(BoostTestTargets) set(base_test_SOURCES - base-array.cpp base-convert.cpp base-dictionary.cpp base-fifo.cpp + base-array.cpp base-base64.cpp base-convert.cpp base-dictionary.cpp base-fifo.cpp base-json.cpp base-match.cpp base-netstring.cpp base-object.cpp base-serialize.cpp base-shellescape.cpp base-stacktrace.cpp base-stream.cpp base-string.cpp base-timer.cpp base-type.cpp base-value.cpp config-ops.cpp icinga-checkresult.cpp icinga-macros.cpp icinga-notification.cpp - icinga-perfdata.cpp remote-base64.cpp remote-url.cpp + icinga-perfdata.cpp remote-url.cpp ) if(ICINGA2_UNITY_BUILD) @@ -44,6 +44,7 @@ add_boost_test(base base_array/foreach base_array/clone base_array/json + base_base64/base64 base_convert/tolong base_convert/todouble base_convert/tostring @@ -112,7 +113,6 @@ add_boost_test(base icinga_perfdata/ignore_invalid_warn_crit_min_max icinga_perfdata/invalid icinga_perfdata/multi - remote_base64/base64 remote_url/id_and_path remote_url/parameters remote_url/get_and_set diff --git a/test/remote-base64.cpp b/test/base-base64.cpp similarity index 97% rename from test/remote-base64.cpp rename to test/base-base64.cpp index 10c95846f..d4d9576af 100644 --- a/test/remote-base64.cpp +++ b/test/base-base64.cpp @@ -17,12 +17,12 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "remote/base64.hpp" +#include "base/base64.hpp" #include using namespace icinga; -BOOST_AUTO_TEST_SUITE(remote_base64) +BOOST_AUTO_TEST_SUITE(base_base64) BOOST_AUTO_TEST_CASE(base64) { From 7c264673d40f837f31b6d75177c278a80f866615 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 11 Sep 2017 17:00:33 +0200 Subject: [PATCH 2/3] ElasticWriter: Add basic auth support for Elasticsearch behind an HTTP proxy refs #5538 --- doc/09-object-types.md | 2 ++ lib/perfdata/elasticwriter.cpp | 25 ++++++++++++++++++++++++- lib/perfdata/elasticwriter.ti | 2 ++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index f3bf882ab..5c11b4af9 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1000,6 +1000,8 @@ Configuration Attributes: enable_send_perfdata | **Optional.** Send parsed performance data metrics for check results. Defaults to `false`. flush_interval | **Optional.** How long to buffer data points before transfering to Elasticsearch. Defaults to `10`. flush_threshold | **Optional.** How many data points to buffer before forcing a transfer to Elasticsearch. Defaults to `1024`. + username | **Optional.** Basic auth username if Elasticsearch is hidden behind an HTTP proxy. + password | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy. Note: If `flush_threshold` is set too low, this will force the feature to flush all data to Elasticsearch too often. Experiment with the setting, if you are processing more than 1024 metrics per second or similar. diff --git a/lib/perfdata/elasticwriter.cpp b/lib/perfdata/elasticwriter.cpp index 667dafbb8..c8f50aba6 100644 --- a/lib/perfdata/elasticwriter.cpp +++ b/lib/perfdata/elasticwriter.cpp @@ -27,6 +27,7 @@ #include "icinga/checkcommand.hpp" #include "base/tcpsocket.hpp" #include "base/stream.hpp" +#include "base/base64.hpp" #include "base/json.hpp" #include "base/utility.hpp" #include "base/networkstream.hpp" @@ -422,12 +423,19 @@ void ElasticWriter::SendRequest(const String& body) req.AddHeader("Accept", "application/json"); req.AddHeader("Content-Type", "application/json"); + /* Send authentication if configured. */ + String username = GetUsername(); + String password = GetPassword(); + + if (!username.IsEmpty() && !password.IsEmpty()) + req.AddHeader("Authorization", "Basic " + Base64::Encode(username + ":" + password)); + req.RequestMethod = "POST"; req.RequestUrl = url; #ifdef I2_DEBUG /* I2_DEBUG */ Log(LogDebug, "ElasticWriter") - << "Sending body: " << body; + << "Sending request" << ((!username.IsEmpty() && !password.IsEmpty()) ? " with basic auth " : "" ) << body; #endif /* I2_DEBUG */ try { @@ -451,6 +459,20 @@ void ElasticWriter::SendRequest(const String& body) } if (resp.StatusCode > 299) { + if (resp.StatusCode == 401) { + /* More verbose error logging with Elasticsearch is hidden behind a proxy. */ + if (!username.IsEmpty() && !password.IsEmpty()) { + Log(LogCritical, "ElasticWriter") + << "401 Unauthorized. Please ensure that the user '" << username + << "' is able to authenticate against the HTTP API/Proxy."; + } else { + Log(LogCritical, "ElasticWriter") + << "401 Unauthorized. The HTTP API requires authentication but no username/password has been configured."; + } + + return; + } + Log(LogWarning, "ElasticWriter") << "Unexpected response code " << resp.StatusCode; @@ -459,6 +481,7 @@ void ElasticWriter::SendRequest(const String& body) resp.Parse(context, true); String contentType = resp.Headers->Get("content-type"); + if (contentType != "application/json") { Log(LogWarning, "ElasticWriter") << "Unexpected Content-Type: " << contentType; diff --git a/lib/perfdata/elasticwriter.ti b/lib/perfdata/elasticwriter.ti index 430bb402c..023d9e349 100644 --- a/lib/perfdata/elasticwriter.ti +++ b/lib/perfdata/elasticwriter.ti @@ -19,6 +19,8 @@ class ElasticWriter : ConfigObject [config] bool enable_send_perfdata { default {{{ return false; }}} }; + [config] String username; + [config] String password; [config] int flush_interval { default {{{ return 10; }}} From d801aaa611b7fb0c3454db2c32bce769edcd0b3d Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 11 Sep 2017 17:28:41 +0200 Subject: [PATCH 3/3] ElasticWriter: Implement support for TLS connections (HTTP proxy) This commit also enhances the log messages. refs #5538 --- doc/09-object-types.md | 16 +++++++++--- doc/14-features.md | 4 ++- lib/perfdata/elasticwriter.cpp | 45 +++++++++++++++++++++++++++------- lib/perfdata/elasticwriter.ti | 7 ++++++ 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index 5c11b4af9..e278baaee 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -997,15 +997,25 @@ Configuration Attributes: host | **Required.** Elasticsearch host address. Defaults to `127.0.0.1`. port | **Required.** Elasticsearch port. Defaults to `9200`. index | **Required.** Elasticsearch index name. Defaults to `icinga2`. - enable_send_perfdata | **Optional.** Send parsed performance data metrics for check results. Defaults to `false`. - flush_interval | **Optional.** How long to buffer data points before transfering to Elasticsearch. Defaults to `10`. - flush_threshold | **Optional.** How many data points to buffer before forcing a transfer to Elasticsearch. Defaults to `1024`. + enable\_send\_perfdata | **Optional.** Send parsed performance data metrics for check results. Defaults to `false`. + flush\_interval | **Optional.** How long to buffer data points before transfering to Elasticsearch. Defaults to `10`. + flush\_threshold | **Optional.** How many data points to buffer before forcing a transfer to Elasticsearch. Defaults to `1024`. username | **Optional.** Basic auth username if Elasticsearch is hidden behind an HTTP proxy. password | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy. + enable\_tls | **Optional.** Whether to use a TLS stream. Defaults to `false`. Requires an HTTP proxy. + ca\_path | **Optional.** CA certificate to validate the remote host. Requires `enable_tls` set to `true`. + cert\_path | **Optional.** Host certificate to present to the remote host for mutual verification. Requires `enable_tls` set to `true`. + key\_path | **Optional.** Host key to accompany the cert\_path. Requires `enable_tls` set to `true`. Note: If `flush_threshold` is set too low, this will force the feature to flush all data to Elasticsearch too often. Experiment with the setting, if you are processing more than 1024 metrics per second or similar. +Basic auth is supported with the `username` and `password` attributes. This requires an +HTTP proxy (Nginx, etc.) in front of the Elasticsearch instance. + +TLS for the HTTP proxy can be enabled with `enable_tls`. In addition to that +you can specify the certificates with the `ca_path`, `cert_path` and `cert_key` attributes. + ## LiveStatusListener Livestatus API interface available as TCP or UNIX socket. Historical table queries diff --git a/doc/14-features.md b/doc/14-features.md index 70c75098e..5b7b5e372 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -288,7 +288,9 @@ The check results include parsed performance data metrics if enabled. Enable the feature and restart Icinga 2. - # icinga2 feature enable elastic +``` +# icinga2 feature enable elastic +``` The default configuration expects an Elasticsearch instance running on `localhost` on port `9200 and writes to an index called `icinga2`. diff --git a/lib/perfdata/elasticwriter.cpp b/lib/perfdata/elasticwriter.cpp index c8f50aba6..83278bea7 100644 --- a/lib/perfdata/elasticwriter.cpp +++ b/lib/perfdata/elasticwriter.cpp @@ -356,12 +356,13 @@ void ElasticWriter::Enqueue(String type, const Dictionary::Ptr& fields, double t * We do it this way to avoid problems with a near full queue. */ - String data; + String indexBody = "{ \"index\" : { \"_type\" : \"" + eventType + "\" } }\n"; + String fieldsBody = JsonEncode(fields); - data += "{ \"index\" : { \"_type\" : \"" + eventType + "\" } }\n"; - data += JsonEncode(fields); + Log(LogDebug, "ElasticWriter") + << "Add to fields to message list: '" << fieldsBody << "'."; - m_DataBuffer.push_back(data); + m_DataBuffer.push_back(indexBody + fieldsBody); /* Flush if we've buffered too much to prevent excessive memory use. */ if (static_cast(m_DataBuffer.size()) >= GetFlushThreshold()) { @@ -400,7 +401,8 @@ void ElasticWriter::Flush(void) void ElasticWriter::SendRequest(const String& body) { Url::Ptr url = new Url(); - url->SetScheme("http"); + + url->SetScheme(GetEnableTls() ? "https" : "http"); url->SetHost(GetHost()); url->SetPort(GetPort()); @@ -433,10 +435,10 @@ void ElasticWriter::SendRequest(const String& body) req.RequestMethod = "POST"; req.RequestUrl = url; -#ifdef I2_DEBUG /* I2_DEBUG */ + /* Don't log the request body to debug log, this is already done above. */ Log(LogDebug, "ElasticWriter") - << "Sending request" << ((!username.IsEmpty() && !password.IsEmpty()) ? " with basic auth " : "" ) << body; -#endif /* I2_DEBUG */ + << "Sending " << req.RequestMethod << " request" << ((!username.IsEmpty() && !password.IsEmpty()) ? " with basic auth" : "" ) + << " to '" << url->Format() << "'."; try { req.WriteBody(body.CStr(), body.GetLength()); @@ -523,7 +525,32 @@ Stream::Ptr ElasticWriter::Connect(void) << "Can't connect to Elasticsearch on host '" << GetHost() << "' port '" << GetPort() << "'."; throw ex; } - return new NetworkStream(socket); + + if (GetEnableTls()) { + boost::shared_ptr sslContext; + + try { + sslContext = MakeSSLContext(GetCertPath(), GetKeyPath(), GetCaPath()); + } catch (const std::exception& ex) { + Log(LogWarning, "ElasticWriter") + << "Unable to create SSL context."; + throw ex; + } + + TlsStream::Ptr tlsStream = new TlsStream(socket, GetHost(), RoleClient, sslContext); + + try { + tlsStream->Handshake(); + } catch (const std::exception& ex) { + Log(LogWarning, "ElasticWriter") + << "TLS handshake with host '" << GetHost() << "' on port " << GetPort() << " failed."; + throw ex; + } + + return tlsStream; + } else { + return new NetworkStream(socket); + } } void ElasticWriter::AssertOnWorkQueue(void) diff --git a/lib/perfdata/elasticwriter.ti b/lib/perfdata/elasticwriter.ti index 023d9e349..cbd8ff170 100644 --- a/lib/perfdata/elasticwriter.ti +++ b/lib/perfdata/elasticwriter.ti @@ -22,6 +22,13 @@ class ElasticWriter : ConfigObject [config] String username; [config] String password; + [config] bool enable_tls { + default {{{ return false; }}} + }; + [config] String ca_path; + [config] String cert_path; + [config] String key_path; + [config] int flush_interval { default {{{ return 10; }}} };