Merge branch 'feature/gelf-writer-7619'

fixes #7619

doc/1-about.md

@@ -62,6 +62,8 @@ Icinga 2 is available as [Vagrant Demo VM](#vagrant).
     * schema files in `lib/db_ido_{mysql,pgsql}/schema` (source)
     * Table `programstatus`: New column `program_version`
     * Table `customvariables` and `customvariablestatus`: New column `is_json` (required for custom attribute array/dictionary support)
+* New features
+    * [GelfWriter](#gelfwriter): Logging check results, state changes, notifications to GELF (graylog2, logstash) #7619
 * New CLI commands #7245
     * `icinga2 feature {enable,disable}` replaces `icinga2-{enable,disable}-feature` script  #7250
     * `icinga2 object list` replaces `icinga2-list-objects` script  #7251
@@ -162,6 +164,12 @@ Icinga 2 still supports writing performance data files for graphing addons, but
 capability of writing performance data directly into a Graphite TCP socket simplifying realtime
 monitoring graphs.
 
+* Native support for writing log events to [GELF](#gelf-writer) receivers (graylog2, Logstash)
+
+Icinga 2 will write all check result, state change and notification event logs into a defined
+[GELF](#gelfwriter) input receiver. Natively provided by [graylog2](http://www.graylog2.org),
+and as additional input type provided by [Logstash](http://logstash.net).
+
 * Dynamic configuration language
 
 Simple [apply](#using-apply) and [assign](#group-assign) rules for creating configuration object

doc/4-monitoring-basics.md

@@ -2171,7 +2171,7 @@ You can enable the feature using
     # icinga2 feature enable graphite
 
 By default the `GraphiteWriter` object expects the Graphite Carbon Cache to listen at
-`127.0.0.1` on port `2003`.
+`127.0.0.1` on TCP port `2003`.
 
 The current naming schema is
 
@@ -2228,6 +2228,29 @@ Cache. Please make sure that the order is correct because the first match wins.
     pattern = ^icinga\.
     retentions = 1m:2d,5m:10d,30m:90d,360m:4y
 
+### <a id="gelfwriter"></a> GELF Writer
+
+The `Graylog Extended Log Format` (short: [GELF](http://www.graylog2.org/resources/gelf))
+can be used to send application logs directly to a TCP socket.
+
+While it has been specified by the [graylog2](http://www.graylog2.org/) project as their
+[input resource standard](http://www.graylog2.org/resources/gelf), other tools such as
+[Logstash](http://www.logstash.net) also support `GELF` as
+[input type](http://logstash.net/docs/latest/inputs/gelf).
+
+You can enable the feature using
+
+    # icinga2 feature enable gelf
+
+By default the `GelfWriter` object expects the GELF receiver to listen at `127.0.0.1` on TCP port `12201`.
+The default `source`  attribute is set to `icinga2`. You can customize that for your needs if required.
+
+Currently these events are processed:
+* Check results
+* State changes
+* Notifications
+
+
 ## <a id="status-data"></a> Status Data
 
 Icinga 1.x writes object configuration data and status data in a cyclic

doc/7-configuring-icinga-2.md

@@ -1335,6 +1335,28 @@ Example with your custom [global constant](#global-constants) `GraphiteEnv`:
     host_name_template = GraphiteEnv + ".$host.name$"
     service_name_template = GraphiteEnv + ".$host.name$.$service.name$"
 
+### <a id="objecttype-gelfwriter"></a> GelfWriter
+
+Writes event log entries to a defined GELF receiver host (Graylog2, Logstash).
+
+Example:
+
+    library "perfdata"
+
+    object GelfWriter "gelf" {
+      host = "127.0.0.1"
+      port = 12201
+    }
+
+Attributes:
+
+  Name            	|Description
+  ----------------------|----------------------
+  host            	|**Optional.** GELF receiver host address. Defaults to '127.0.0.1'.
+  port            	|**Optional.** GELF receiver port. Defaults to `12201`.
+  source		|**Optional.** Source name for this instance. Defaults to `icinga2`.
+
+
 ### <a id="objecttype-idomysqlconnection"></a> IdoMySqlConnection
 
 IDO database adapter for MySQL.

etc/CMakeLists.txt

@@ -41,6 +41,7 @@ install_if_not_exists(icinga2/features-available/checker.conf ${CMAKE_INSTALL_SY
 install_if_not_exists(icinga2/features-available/command.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
 install_if_not_exists(icinga2/features-available/compatlog.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
 install_if_not_exists(icinga2/features-available/debuglog.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
+install_if_not_exists(icinga2/features-available/gelf.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
 install_if_not_exists(icinga2/features-available/graphite.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
 install_if_not_exists(icinga2/features-available/icingastatus.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)
 install_if_not_exists(icinga2/features-available/ido-mysql.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available)

etc/icinga2/features-available/gelf.conf

@@ -0,0 +1,12 @@
+/**
+ * The GelfWriter type writes event log entries
+ * to a GELF tcp socket provided by graylog2,
+ * logstash or any other receiver.
+ */
+
+library "perfdata"
+
+object GelfWriter "gelf" {
+  //host = "127.0.0.1"
+  //port = 12201
+}

lib/perfdata/CMakeLists.txt

@@ -15,13 +15,14 @@
 # along with this program; if not, write to the Free Software Foundation
 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
 
+mkclass_target(gelfwriter.ti gelfwriter.thpp)
 mkclass_target(graphitewriter.ti graphitewriter.thpp)
 mkclass_target(perfdatawriter.ti perfdatawriter.thpp)
 
 mkembedconfig_target(perfdata-type.conf perfdata-type.cpp)
 
 set(perfdata_SOURCES
-  graphitewriter.cpp graphitewriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp
+  gelfwriter.cpp gelfwriter.thpp graphitewriter.cpp graphitewriter.thpp perfdatawriter.cpp perfdatawriter.thpp perfdata-type.cpp
 )
 
 if(ICINGA2_UNITY_BUILD)

lib/perfdata/gelfwriter.cpp

@@ -0,0 +1,204 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org)    *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#include "perfdata/gelfwriter.hpp"
+#include "icinga/service.hpp"
+#include "icinga/notification.hpp"
+#include "icinga/macroprocessor.hpp"
+#include "icinga/compatutility.hpp"
+#include "icinga/perfdatavalue.hpp"
+#include "base/tcpsocket.hpp"
+#include "base/dynamictype.hpp"
+#include "base/objectlock.hpp"
+#include "base/logger.hpp"
+#include "base/utility.hpp"
+#include "base/stream.hpp"
+#include "base/networkstream.hpp"
+#include "base/json.hpp"
+
+using namespace icinga;
+
+REGISTER_TYPE(GelfWriter);
+
+void GelfWriter::Start(void)
+{
+	DynamicObject::Start();
+
+	m_ReconnectTimer = new Timer();
+	m_ReconnectTimer->SetInterval(10);
+	m_ReconnectTimer->OnTimerExpired.connect(boost::bind(&GelfWriter::ReconnectTimerHandler, this));
+	m_ReconnectTimer->Start();
+	m_ReconnectTimer->Reschedule(0);
+
+	// Send check results
+	Service::OnNewCheckResult.connect(boost::bind(&GelfWriter::CheckResultHandler, this, _1, _2));
+	// Send notifications
+	Service::OnNotificationSentToUser.connect(boost::bind(&GelfWriter::NotificationToUserHandler, this, _1, _2, _3, _4, _5, _6, _7, _8));
+	// Send state change
+	Service::OnStateChange.connect(boost::bind(&GelfWriter::StateChangeHandler, this, _1, _2, _3));
+}
+
+void GelfWriter::ReconnectTimerHandler(void)
+{
+	if (m_Stream)
+		return;
+
+	TcpSocket::Ptr socket = new TcpSocket();
+
+	Log(LogNotice, "GelfWriter")
+	    << "Reconnecting to GELF endpoint '" << GetHost() << "' port '" << GetPort() << "'.";
+
+	try {
+		socket->Connect(GetHost(), GetPort());
+	} catch (std::exception&) {
+		Log(LogCritical, "GelfWriter")
+		    << "Can't connect to GELF endpoint '" << GetHost() << "' port '" << GetPort() << "'.";
+		return;
+	}
+
+	m_Stream = new NetworkStream(socket);
+}
+
+void GelfWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
+{
+	CONTEXT("GELF Processing check result for '" + checkable->GetName() + "'");
+
+	Log(LogDebug, "GelfWriter")
+	    << "GELF Processing check result for '" << checkable->GetName() << "'";
+
+	Dictionary::Ptr fields = new Dictionary();
+	Service::Ptr service = dynamic_pointer_cast<Service>(checkable);
+	Host::Ptr host;
+
+	if (service) {
+		host = service->GetHost();
+		fields->Set("_service_name", service->GetShortName());
+		fields->Set("_service_state", Service::StateToString(service->GetState()));
+	} else {
+		host = static_pointer_cast<Host>(checkable);
+	}
+	fields->Set("_hostname", host->GetName());
+	fields->Set("short_message", cr->GetOutput());
+	fields->Set("_type", "CHECK RESULT");
+
+	SendLogMessage(ComposeGelfMessage(fields, GetSource()));
+}
+
+void GelfWriter::NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
+    const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
+    const String& author, const String& comment_text, const String& command_name)
+{
+  CONTEXT("GELF Processing notification to all users '" + checkable->GetName() + "'");
+
+	Host::Ptr host;
+	Service::Ptr service;
+	tie(host, service) = GetHostService(checkable);
+
+	String notification_type_str = Notification::NotificationTypeToString(notification_type);
+
+	String author_comment = "";
+	if (notification_type == NotificationCustom || notification_type == NotificationAcknowledgement) {
+		author_comment = author + ";" + comment_text;
+	}
+
+	String output;
+	if (cr)
+		output = CompatUtility::GetCheckResultOutput(cr);
+
+	Dictionary::Ptr fields = new Dictionary();
+	if (service) {
+		host = service->GetHost();
+		fields->Set("_type", "SERVICE NOTIFICATION");
+		fields->Set("_service", service->GetShortName());
+		fields->Set("short_message", output);
+	} else {
+		host = static_pointer_cast<Host>(checkable);
+		fields->Set("_type", "HOST NOTIFICATION");
+		fields->Set("short_message", "(" << (host->IsReachable() ? Host::StateToString(host->GetState()) : "UNREACHABLE") << ")");
+	}
+	fields->Set("_hostname", host->GetName());
+	fields->Set("_command", command_name);
+	fields->Set("_state", notification_type_str);
+	fields->Set("_comment", author_comment);
+
+	SendLogMessage(ComposeGelfMessage(fields, GetSource()));
+}
+
+void GelfWriter::StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
+{
+	CONTEXT("GELF Processing state change '" + checkable->GetName() + "'");
+
+	Host::Ptr host;
+	Service::Ptr service;
+	tie(host, service) = GetHostService(checkable);
+
+	Dictionary::Ptr fields = new Dictionary();
+	fields->Set("_state", service ? static_cast<int>(service->GetState()) : static_cast<int>(host->GetState()));
+	fields->Set("_type", "STATE CHANGE");
+	fields->Set("_current_check_attempt", checkable->GetCheckAttempt());
+	fields->Set("_max_check_attempts", checkable->GetMaxCheckAttempts());
+
+	if (service) {
+		fields->Set("_last_state", service->GetLastState());
+		fields->Set("_last_hard_state", service->GetLastHardState());
+	} else {
+		fields->Set("_last_state", host->GetLastState());
+		fields->Set("_last_hard_state", host->GetLastHardState());
+	}
+
+	if (cr) {
+		fields->Set("short_message", CompatUtility::GetCheckResultOutput(cr));
+		fields->Set("full_message", CompatUtility::GetCheckResultLongOutput(cr));
+		fields->Set("_check_source", cr->GetCheckSource());
+	}
+
+	SendLogMessage(ComposeGelfMessage(fields, GetSource()));
+}
+
+String GelfWriter::ComposeGelfMessage(const Dictionary::Ptr& fields, const String& source)
+{
+	fields->Set("version", "1.1");
+	fields->Set("host", source);
+	fields->Set("timestamp", Utility::GetTime());
+
+	return JsonEncode(fields);
+}
+
+void GelfWriter::SendLogMessage(const String& gelf)
+{
+	std::ostringstream msgbuf;
+	msgbuf << gelf;
+	msgbuf << '\0';
+
+	String log = msgbuf.str();
+
+	ObjectLock olock(this);
+
+	if (!m_Stream)
+		return;
+
+	try {
+		m_Stream->Write(log.CStr(), log.GetLength());
+	} catch (const std::exception& ex) {
+		Log(LogCritical, "GelfWriter")
+		    << "Cannot write to TCP socket on host '" << GetHost() << "' port '" << GetPort() << "'.";
+
+		m_Stream.reset();
+	}
+}

lib/perfdata/gelfwriter.hpp

@@ -0,0 +1,65 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org)    *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#ifndef GELFWRITER_H
+#define GELFWRITER_H
+
+#include "perfdata/gelfwriter.thpp"
+#include "icinga/service.hpp"
+#include "base/dynamicobject.hpp"
+#include "base/tcpsocket.hpp"
+#include "base/timer.hpp"
+#include <fstream>
+
+namespace icinga
+{
+
+/**
+ * An Icinga gelf writer.
+ *
+ * @ingroup perfdata
+ */
+class GelfWriter : public ObjectImpl<GelfWriter>
+{
+public:
+	DECLARE_OBJECT(GelfWriter);
+	DECLARE_OBJECTNAME(GelfWriter);
+
+protected:
+	virtual void Start(void);
+
+private:
+	Stream::Ptr m_Stream;
+	
+	Timer::Ptr m_ReconnectTimer;
+
+	void CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
+  void NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
+    const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
+    const String& author, const String& comment_text, const String& command_name);
+  String ComposeGelfMessage(const Dictionary::Ptr& fields, const String& source);
+  void StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type);
+  void SendLogMessage(const String& gelf);
+
+	void ReconnectTimerHandler(void);
+};
+
+}
+
+#endif /* GELFWRITER_H */

lib/perfdata/gelfwriter.ti

@@ -0,0 +1,38 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012-2014 Icinga Development Team (http://www.icinga.org)    *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#include "base/dynamicobject.hpp"
+
+namespace icinga
+{
+
+class GelfWriter : DynamicObject
+{
+	[config] String host {
+		default {{{ return "127.0.0.1"; }}}
+	};
+	[config] String port {
+		default {{{ return "12201"; }}}
+	};
+	[config] String source {
+		default {{{ return "icinga2"; }}}
+	};
+};
+
+}

lib/perfdata/perfdata-type.conf

@@ -33,3 +33,10 @@
 	%attribute %string "host_name_template",
 	%attribute %string "service_name_template"
 }
+
+%type GelfWriter {
+        %attribute %string "host",
+        %attribute %string "port",
+        %attribute %string "source"
+}
+