Implement flapping detection.

lib/icinga/Makefile.am

@@ -62,6 +62,7 @@ libicinga_la_SOURCES =  \
 	service-comment.cpp \
 	service-downtime.cpp \
 	service-event.cpp \
+	service-flapping.cpp \
 	service-notification.cpp \
 	service.h \
 	servicegroup.cpp \

lib/icinga/notification.cpp

@@ -204,6 +204,10 @@ String Notification::NotificationTypeToString(NotificationType type)
 			return "PROBLEM";
 		case NotificationRecovery:
 			return "RECOVERY";
+		case NotificationFlappingStart:
+			return "FLAPPINGSTART";
+		case NotificationFlappingEnd:
+			return "FLAPPINGEND";
 		default:
 			return "UNKNOWN_NOTIFICATION";
 	}

lib/icinga/notification.h

@@ -42,7 +42,9 @@ enum NotificationType
 	NotificationCustom,
 	NotificationAcknowledgement,
 	NotificationProblem,
-	NotificationRecovery
+	NotificationRecovery,
+	NotificationFlappingStart,
+	NotificationFlappingEnd,
 };
 
 class Service;

lib/icinga/service-check.cpp

@@ -367,8 +367,8 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 	SetState(static_cast<ServiceState>(state));
 
 	bool call_eventhandler = false;
-
-	if (old_state != GetState()) {
+	bool stateChange = (old_state != GetState());
+	if (stateChange) {
 		SetLastStateChange(now);
 
 		/* remove acknowledgements */
@@ -449,6 +449,13 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 
 	olock.Lock();
 	SetLastCheckResult(cr);
+
+	bool was_flapping, is_flapping;
+
+	was_flapping = IsFlapping();
+	UpdateFlappingStatus(stateChange);
+	is_flapping = IsFlapping();
+
 	olock.Unlock();
 
 	/* Flush the object so other instances see the service's
@@ -473,7 +480,11 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 	if (send_downtime_notification)
 		RequestNotifications(in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr);
 
-	if (send_notification)
+	if (!was_flapping && is_flapping)
+		RequestNotifications(NotificationFlappingStart, cr);
+	else if (was_flapping && !is_flapping)
+		RequestNotifications(NotificationFlappingEnd, cr);
+	else if (send_notification)
 		RequestNotifications(recovery ? NotificationRecovery : NotificationProblem, cr);
 }
 

lib/icinga/service-flapping.cpp

@@ -0,0 +1,78 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012 Icinga Development Team (http://www.icinga.org/)        *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#include "icinga/service.h"
+#include "base/dynamictype.h"
+#include "base/objectlock.h"
+#include "base/logger_fwd.h"
+#include "base/timer.h"
+#include "base/utility.h"
+#include <boost/tuple/tuple.hpp>
+#include <boost/smart_ptr/make_shared.hpp>
+#include <boost/foreach.hpp>
+
+using namespace icinga;
+
+#define FLAPPING_INTERVAL (30 * 60)
+
+void Service::UpdateFlappingStatus(bool stateChange)
+{
+	double ts, now;
+	long counter;
+
+	now = Utility::GetTime();
+
+	if (m_FlappingLastChange.IsEmpty()) {
+		ts = now;
+		counter = 0;
+	} else {
+		ts = m_FlappingLastChange;
+		counter = m_FlappingCounter;
+	}
+
+	double diff = now - ts;
+
+	if (diff > 0)
+		counter -= 0.5 * m_FlappingCounter / (diff / FLAPPING_INTERVAL);
+
+	if (stateChange)
+		counter += diff;
+
+	m_FlappingCounter = counter;
+	Touch("flapping_counter");
+
+	m_FlappingLastChange = now;
+	Touch("flapping_lastchange");
+}
+
+bool Service::IsFlapping(void) const
+{
+	double threshold = 30;
+
+	if (!m_FlappingThreshold.IsEmpty())
+		threshold = m_FlappingThreshold;
+
+	if (m_FlappingCounter.IsEmpty())
+		return false;
+
+	long counter = m_FlappingCounter;
+
+	return (counter > threshold * FLAPPING_INTERVAL / 100);
+
+}

lib/icinga/service.cpp

@@ -83,6 +83,10 @@ Service::Service(const Dictionary::Ptr& serializedObject)
 	RegisterAttribute("enable_notifications", Attribute_Replicated, &m_EnableNotifications);
 	RegisterAttribute("force_next_notification", Attribute_Replicated, &m_ForceNextNotification);
 
+	RegisterAttribute("flapping_counter", Attribute_Replicated, &m_FlappingCounter);
+	RegisterAttribute("flapping_lastchange", Attribute_Replicated, &m_FlappingLastChange);
+	RegisterAttribute("flapping_threshold", Attribute_Config, &m_FlappingThreshold);
+
 	SetSchedulingOffset(rand());
 }
 

lib/icinga/service.h

@@ -248,6 +248,10 @@ public:
 	void ExecuteEventHandler(void);
 	shared_ptr<EventCommand> GetEventCommand(void) const;
 
+	/* Flapping Detection */
+	bool IsFlapping(void) const;
+	void UpdateFlappingStatus(bool stateChange);
+
 protected:
 	virtual void OnRegistrationCompleted(void);
 	virtual void OnAttributeChanged(const String& name);
@@ -319,6 +323,11 @@ private:
 
 	/* Event Handler */
 	Attribute<String> m_EventCommand;
+
+	/* Flapping */
+	Attribute<long> m_FlappingCounter;
+	Attribute<double> m_FlappingLastChange;
+	Attribute<double> m_FlappingThreshold;
 };
 
 }