From c71029f2e8e6a3eaf048267bfc42f3ece978228d Mon Sep 17 00:00:00 2001
From: Julian Brost <julian.brost@icinga.com>
Date: Wed, 8 Dec 2021 11:49:42 +0100
Subject: [PATCH] Set downtime trigger time deterministically

When triggering a downtime, the time of the causing event is now passed on as
the trigger time. That time is:

* For fixed downtimes: the later one of start and entry time.
* If a check result triggers the downtime: The execution end of the check
  result.
* If another downtime triggers the downtime: The trigger time of the first
  downtime.

This is done so two nodes in a HA setup can write consistent Icinga DB downtime
history streams.

refs #9101
---
 lib/icinga/checkable-check.cpp    |  2 +-
 lib/icinga/checkable-downtime.cpp |  4 ++--
 lib/icinga/checkable.hpp          |  2 +-
 lib/icinga/downtime.cpp           | 16 +++++++++-------
 lib/icinga/downtime.hpp           |  2 +-
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp
index 782899990..71d3cb13e 100644
--- a/lib/icinga/checkable-check.cpp
+++ b/lib/icinga/checkable-check.cpp
@@ -327,7 +327,7 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig
 	}
 
 	if (!IsStateOK(new_state))
-		TriggerDowntimes();
+		TriggerDowntimes(cr->GetExecutionEnd());
 
 	/* statistics for external tools */
 	Checkable::UpdateStatistics(cr, checkableType);
diff --git a/lib/icinga/checkable-downtime.cpp b/lib/icinga/checkable-downtime.cpp
index 3729c27ab..d96003d46 100644
--- a/lib/icinga/checkable-downtime.cpp
+++ b/lib/icinga/checkable-downtime.cpp
@@ -16,10 +16,10 @@ void Checkable::RemoveAllDowntimes()
 	}
 }
 
-void Checkable::TriggerDowntimes()
+void Checkable::TriggerDowntimes(double triggerTime)
 {
 	for (const Downtime::Ptr& downtime : GetDowntimes()) {
-		downtime->TriggerDowntime();
+		downtime->TriggerDowntime(triggerTime);
 	}
 }
 
diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp
index 46916ddf0..292dac281 100644
--- a/lib/icinga/checkable.hpp
+++ b/lib/icinga/checkable.hpp
@@ -139,7 +139,7 @@ public:
 	int GetDowntimeDepth() const final;
 
 	void RemoveAllDowntimes();
-	void TriggerDowntimes();
+	void TriggerDowntimes(double triggerTime);
 	bool IsInDowntime() const;
 	bool IsAcknowledged() const;
 
diff --git a/lib/icinga/downtime.cpp b/lib/icinga/downtime.cpp
index a9d23731b..ca85f0fce 100644
--- a/lib/icinga/downtime.cpp
+++ b/lib/icinga/downtime.cpp
@@ -9,6 +9,7 @@
 #include "base/utility.hpp"
 #include "base/timer.hpp"
 #include <boost/thread/once.hpp>
+#include <cmath>
 
 using namespace icinga;
 
@@ -130,7 +131,7 @@ void Downtime::Start(bool runtimeCreated)
 		Log(LogNotice, "Downtime")
 			<< "Checkable '" << checkable->GetName() << "' already in a NOT-OK state."
 			<< " Triggering downtime now.";
-		TriggerDowntime();
+		TriggerDowntime(checkable->GetLastStateChange());
 	}
 
 	if (GetFixed() && CanBeTriggered()) {
@@ -138,7 +139,7 @@ void Downtime::Start(bool runtimeCreated)
 		OnDowntimeStarted(this);
 
 		/* Trigger fixed downtime immediately. */
-		TriggerDowntime();
+		TriggerDowntime(std::fmax(GetStartTime(), GetEntryTime()));
 	}
 }
 
@@ -422,7 +423,7 @@ bool Downtime::CanBeTriggered()
 	return true;
 }
 
-void Downtime::TriggerDowntime()
+void Downtime::TriggerDowntime(double triggerTime)
 {
 	if (!CanBeTriggered())
 		return;
@@ -432,8 +433,9 @@ void Downtime::TriggerDowntime()
 	Log(LogInformation, "Downtime")
 		<< "Triggering downtime '" << GetName() << "' for checkable '" << checkable->GetName() << "'.";
 
-	if (GetTriggerTime() == 0)
-		SetTriggerTime(Utility::GetTime());
+	if (GetTriggerTime() == 0) {
+		SetTriggerTime(triggerTime);
+	}
 
 	Array::Ptr triggers = GetTriggers();
 
@@ -445,7 +447,7 @@ void Downtime::TriggerDowntime()
 			if (!downtime)
 				continue;
 
-			downtime->TriggerDowntime();
+			downtime->TriggerDowntime(triggerTime);
 		}
 	}
 
@@ -475,7 +477,7 @@ void Downtime::DowntimesStartTimerHandler()
 			OnDowntimeStarted(downtime);
 
 			/* Trigger fixed downtime immediately. */
-			downtime->TriggerDowntime();
+			downtime->TriggerDowntime(std::fmax(downtime->GetStartTime(), downtime->GetEntryTime()));
 		}
 	}
 }
diff --git a/lib/icinga/downtime.hpp b/lib/icinga/downtime.hpp
index e99e20ed7..9c0368cb4 100644
--- a/lib/icinga/downtime.hpp
+++ b/lib/icinga/downtime.hpp
@@ -57,7 +57,7 @@ public:
 	void UnregisterChild(const Downtime::Ptr& downtime);
 	std::set<Downtime::Ptr> GetChildren() const;
 
-	void TriggerDowntime();
+	void TriggerDowntime(double triggerTime);
 
 	static String GetDowntimeIDFromLegacyID(int id);