Merge pull request #5569 from Icinga/feature/elastic-writer-http-proxy

ElasticWriter: Add basic auth and TLS support for Elasticsearch behind an HTTP proxy

doc/09-object-types.md

@@ -997,13 +997,25 @@ Configuration Attributes:
   host                   | **Required.** Elasticsearch host address. Defaults to `127.0.0.1`.
   port                   | **Required.** Elasticsearch port. Defaults to `9200`.
   index                  | **Required.** Elasticsearch index name. Defaults to `icinga2`.
-  enable_send_perfdata   | **Optional.** Send parsed performance data metrics for check results. Defaults to `false`.
+  enable\_send\_perfdata | **Optional.** Send parsed performance data metrics for check results. Defaults to `false`.
-  flush_interval         | **Optional.** How long to buffer data points before transfering to Elasticsearch. Defaults to `10`.
+  flush\_interval        | **Optional.** How long to buffer data points before transfering to Elasticsearch. Defaults to `10`.
-  flush_threshold        | **Optional.** How many data points to buffer before forcing a transfer to Elasticsearch.  Defaults to `1024`.
+  flush\_threshold       | **Optional.** How many data points to buffer before forcing a transfer to Elasticsearch.  Defaults to `1024`.
+  username               | **Optional.** Basic auth username if Elasticsearch is hidden behind an HTTP proxy.
+  password               | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy.
+  enable\_tls            | **Optional.** Whether to use a TLS stream. Defaults to `false`. Requires an HTTP proxy.
+  ca\_path               | **Optional.** CA certificate to validate the remote host. Requires `enable_tls` set to `true`.
+  cert\_path             | **Optional.** Host certificate to present to the remote host for mutual verification. Requires `enable_tls` set to `true`.
+  key\_path              | **Optional.** Host key to accompany the cert\_path. Requires `enable_tls` set to `true`.
 
 Note: If `flush_threshold` is set too low, this will force the feature to flush all data to Elasticsearch too often.
 Experiment with the setting, if you are processing more than 1024 metrics per second or similar.
 
+Basic auth is supported with the `username` and `password` attributes. This requires an
+HTTP proxy (Nginx, etc.) in front of the Elasticsearch instance.
+
+TLS for the HTTP proxy can be enabled with `enable_tls`. In addition to that
+you can specify the certificates with the `ca_path`, `cert_path` and `cert_key` attributes.
+
 ## LiveStatusListener <a id="objecttype-livestatuslistener"></a>
 
 Livestatus API interface available as TCP or UNIX socket. Historical table queries

doc/14-features.md

@@ -288,7 +288,9 @@ The check results include parsed performance data metrics if enabled.
 
 Enable the feature and restart Icinga 2.
 
-    # icinga2 feature enable elastic
+```
+# icinga2 feature enable elastic
+```
 
 The default configuration expects an Elasticsearch instance running on `localhost` on port `9200
  and writes to an index called `icinga2`.

lib/base/CMakeLists.txt

@@ -27,7 +27,7 @@ mkclass_target(sysloglogger.ti sysloglogger.tcpp sysloglogger.thpp)
 
 set(base_SOURCES
   application.cpp application.thpp application-version.cpp array.cpp
-  array-script.cpp boolean.cpp boolean-script.cpp console.cpp context.cpp
+  array-script.cpp boolean.cpp boolean-script.cpp base64.cpp console.cpp context.cpp
   convert.cpp datetime.cpp datetime.thpp datetime-script.cpp debuginfo.cpp dictionary.cpp dictionary-script.cpp
   configobject.cpp configobject.thpp configobject-script.cpp configtype.cpp configwriter.cpp dependencygraph.cpp
   exception.cpp fifo.cpp filelogger.cpp filelogger.thpp initialize.cpp json.cpp

lib/base/base64.cpp

@@ -17,7 +17,7 @@
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
  ******************************************************************************/
 
-#include "remote/base64.hpp"
+#include "base/base64.hpp"
 #include <openssl/bio.h>
 #include <openssl/evp.h>
 #include <openssl/buffer.h>

lib/base/base64.hpp

@@ -31,7 +31,7 @@ namespace icinga
  *
  * @ingroup remote
  */
-struct I2_REMOTE_API Base64
+struct I2_BASE_API Base64
 {
 	static String Decode(const String& data);
 	static String Encode(const String& data);

lib/perfdata/elasticwriter.cpp

@@ -27,6 +27,7 @@
 #include "icinga/checkcommand.hpp"
 #include "base/tcpsocket.hpp"
 #include "base/stream.hpp"
+#include "base/base64.hpp"
 #include "base/json.hpp"
 #include "base/utility.hpp"
 #include "base/networkstream.hpp"
@@ -355,12 +356,13 @@ void ElasticWriter::Enqueue(String type, const Dictionary::Ptr& fields, double t
 	 * We do it this way to avoid problems with a near full queue.
 	 */
 
-	String data;
+	String indexBody = "{ \"index\" : { \"_type\" : \"" + eventType + "\" } }\n";
+	String fieldsBody = JsonEncode(fields);
 
-	data += "{ \"index\" : { \"_type\" : \"" + eventType + "\" } }\n";
+	Log(LogDebug, "ElasticWriter")
-	data += JsonEncode(fields);
+	    << "Add to fields to message list: '" << fieldsBody << "'.";
 
-	m_DataBuffer.push_back(data);
+	m_DataBuffer.push_back(indexBody + fieldsBody);
 
 	/* Flush if we've buffered too much to prevent excessive memory use. */
 	if (static_cast<int>(m_DataBuffer.size()) >= GetFlushThreshold()) {
@@ -399,7 +401,8 @@ void ElasticWriter::Flush(void)
 void ElasticWriter::SendRequest(const String& body)
 {
 	Url::Ptr url = new Url();
-	url->SetScheme("http");
+
+	url->SetScheme(GetEnableTls() ? "https" : "http");
 	url->SetHost(GetHost());
 	url->SetPort(GetPort());
 
@@ -422,13 +425,20 @@ void ElasticWriter::SendRequest(const String& body)
 	req.AddHeader("Accept", "application/json");
 	req.AddHeader("Content-Type", "application/json");
 
+	/* Send authentication if configured. */
+	String username = GetUsername();
+	String password = GetPassword();
+
+	if (!username.IsEmpty() && !password.IsEmpty())
+		req.AddHeader("Authorization", "Basic " + Base64::Encode(username + ":" + password));
+
 	req.RequestMethod = "POST";
 	req.RequestUrl = url;
 
-#ifdef I2_DEBUG /* I2_DEBUG */
+	/* Don't log the request body to debug log, this is already done above. */
 	Log(LogDebug, "ElasticWriter")
-	    << "Sending body: " << body;
+	    << "Sending " << req.RequestMethod << " request" << ((!username.IsEmpty() && !password.IsEmpty()) ? " with basic auth" : "" )
-#endif /* I2_DEBUG */
+	    << " to '" << url->Format() << "'.";
 
 	try {
 		req.WriteBody(body.CStr(), body.GetLength());
@@ -451,6 +461,20 @@ void ElasticWriter::SendRequest(const String& body)
 	}
 
 	if (resp.StatusCode > 299) {
+		if (resp.StatusCode == 401) {
+			/* More verbose error logging with Elasticsearch is hidden behind a proxy. */
+			if (!username.IsEmpty() && !password.IsEmpty()) {
+				Log(LogCritical, "ElasticWriter")
+				    << "401 Unauthorized. Please ensure that the user '" << username
+				    << "' is able to authenticate against the HTTP API/Proxy.";
+			} else {
+				Log(LogCritical, "ElasticWriter")
+				    << "401 Unauthorized. The HTTP API requires authentication but no username/password has been configured.";
+			}
+
+			return;
+		}
+
 		Log(LogWarning, "ElasticWriter")
 		    << "Unexpected response code " << resp.StatusCode;
 
@@ -459,6 +483,7 @@ void ElasticWriter::SendRequest(const String& body)
 			resp.Parse(context, true);
 
 		String contentType = resp.Headers->Get("content-type");
+
 		if (contentType != "application/json") {
 			Log(LogWarning, "ElasticWriter")
 			    << "Unexpected Content-Type: " << contentType;
@@ -500,7 +525,32 @@ Stream::Ptr ElasticWriter::Connect(void)
 		    << "Can't connect to Elasticsearch on host '" << GetHost() << "' port '" << GetPort() << "'.";
 		throw ex;
 	}
+
+	if (GetEnableTls()) {
+		boost::shared_ptr<SSL_CTX> sslContext;
+
+		try {
+			sslContext = MakeSSLContext(GetCertPath(), GetKeyPath(), GetCaPath());
+		} catch (const std::exception& ex) {
+			Log(LogWarning, "ElasticWriter")
+			    << "Unable to create SSL context.";
+			throw ex;
+		}
+
+		TlsStream::Ptr tlsStream = new TlsStream(socket, GetHost(), RoleClient, sslContext);
+
+		try {
+			tlsStream->Handshake();
+		} catch (const std::exception& ex) {
+			Log(LogWarning, "ElasticWriter")
+			    << "TLS handshake with host '" << GetHost() << "' on port " << GetPort() << " failed.";
+			throw ex;
+		}
+
+		return tlsStream;
+	} else {
 		return new NetworkStream(socket);
+	}
 }
 
 void ElasticWriter::AssertOnWorkQueue(void)

lib/perfdata/elasticwriter.ti

@@ -19,6 +19,15 @@ class ElasticWriter : ConfigObject
 	[config] bool enable_send_perfdata {
 		default {{{ return false; }}}
 	};
+	[config] String username;
+	[config] String password;
+
+	[config] bool enable_tls {
+		default {{{ return false; }}}
+	};
+	[config] String ca_path;
+	[config] String cert_path;
+	[config] String key_path;
 
 	[config] int flush_interval {
 		default {{{ return 10; }}}

lib/remote/CMakeLists.txt

@@ -23,7 +23,7 @@ mkclass_target(zone.ti zone.tcpp zone.thpp)
 set(remote_SOURCES
   actionshandler.cpp apiaction.cpp apiclient.cpp
   apifunction.cpp apilistener.cpp apilistener.thpp apilistener-configsync.cpp
-  apilistener-filesync.cpp apiuser.cpp apiuser.thpp authority.cpp base64.cpp
+  apilistener-filesync.cpp apiuser.cpp apiuser.thpp authority.cpp
   consolehandler.cpp configfileshandler.cpp configpackageshandler.cpp configpackageutility.cpp configobjectutility.cpp
   configstageshandler.cpp createobjecthandler.cpp deleteobjecthandler.cpp
   endpoint.cpp endpoint.thpp eventshandler.cpp eventqueue.cpp filterutility.cpp

lib/remote/apiclient.cpp

@@ -18,7 +18,7 @@
  ******************************************************************************/
 
 #include "remote/apiclient.hpp"
-#include "remote/base64.hpp"
+#include "base/base64.hpp"
 #include "base/json.hpp"
 #include "base/logger.hpp"
 #include "base/exception.hpp"

lib/remote/httpclientconnection.cpp

@@ -18,9 +18,9 @@
  ******************************************************************************/
 
 #include "remote/httpclientconnection.hpp"
-#include "remote/base64.hpp"
 #include "base/configtype.hpp"
 #include "base/objectlock.hpp"
+#include "base/base64.hpp"
 #include "base/utility.hpp"
 #include "base/logger.hpp"
 #include "base/exception.hpp"

lib/remote/httpserverconnection.cpp

@@ -23,7 +23,7 @@
 #include "remote/apilistener.hpp"
 #include "remote/apifunction.hpp"
 #include "remote/jsonrpc.hpp"
-#include "remote/base64.hpp"
+#include "base/base64.hpp"
 #include "base/configtype.hpp"
 #include "base/objectlock.hpp"
 #include "base/utility.hpp"

test/CMakeLists.txt

@@ -20,13 +20,13 @@ set(Boost_USE_STATIC_LIBS OFF)
 include(BoostTestTargets)
 
 set(base_test_SOURCES
-  base-array.cpp base-convert.cpp base-dictionary.cpp base-fifo.cpp
+  base-array.cpp base-base64.cpp base-convert.cpp base-dictionary.cpp base-fifo.cpp
   base-json.cpp base-match.cpp base-netstring.cpp base-object.cpp
   base-serialize.cpp base-shellescape.cpp base-stacktrace.cpp
   base-stream.cpp base-string.cpp base-timer.cpp base-type.cpp
   base-value.cpp config-ops.cpp icinga-checkresult.cpp icinga-macros.cpp
   icinga-notification.cpp
-  icinga-perfdata.cpp remote-base64.cpp remote-url.cpp
+  icinga-perfdata.cpp remote-url.cpp
 )
 
 if(ICINGA2_UNITY_BUILD)
@@ -44,6 +44,7 @@ add_boost_test(base
         base_array/foreach
         base_array/clone
         base_array/json
+        base_base64/base64
         base_convert/tolong
         base_convert/todouble
         base_convert/tostring
@@ -112,7 +113,6 @@ add_boost_test(base
         icinga_perfdata/ignore_invalid_warn_crit_min_max
         icinga_perfdata/invalid
         icinga_perfdata/multi
-        remote_base64/base64
         remote_url/id_and_path
         remote_url/parameters
         remote_url/get_and_set

test/base-base64.cpp

@@ -17,12 +17,12 @@
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
  ******************************************************************************/
 
-#include "remote/base64.hpp"
+#include "base/base64.hpp"
 #include <BoostTestTargetConfig.h>
 
 using namespace icinga;
 
-BOOST_AUTO_TEST_SUITE(remote_base64)
+BOOST_AUTO_TEST_SUITE(base_base64)
 
 BOOST_AUTO_TEST_CASE(base64)
 {