Repair broken API config packages at runtime

This means a new timer which checks every 5m whether the
active-stage can be read, and if not, it overwrites the
file on disk with the details from memory.

lib/remote/apilistener.cpp

@@ -271,6 +271,11 @@ void ApiListener::Start(bool runtimeCreated)
 	m_CleanupCertificateRequestsTimer->Start();
 	m_CleanupCertificateRequestsTimer->Reschedule(0);
 
+	m_ApiPackageIntegrityTimer = new Timer();
+	m_ApiPackageIntegrityTimer->OnTimerExpired.connect(std::bind(&ApiListener::CheckApiPackageIntegrity, this));
+	m_ApiPackageIntegrityTimer->SetInterval(300);
+	m_ApiPackageIntegrityTimer->Start();
+
 	OnMasterChanged(true);
 }
 
@@ -1563,6 +1568,27 @@ void ApiListener::UpdateActivePackageStagesCache()
 	}
 }
 
+void ApiListener::CheckApiPackageIntegrity()
+{
+	boost::mutex::scoped_lock lock(m_ActivePackageStagesLock);
+
+	for (auto package : ConfigPackageUtility::GetPackages()) {
+		String activeStage;
+		try {
+			activeStage = ConfigPackageUtility::GetActiveStageFromFile(package);
+		} catch (const std::exception& ex) {
+			/* An error means that the stage is broken, try to repair it. */
+			String activeStageCached = m_ActivePackageStages[package];
+
+			Log(LogInformation, "ApiListener")
+				<< "Repairing broken API config package '" << package
+				<< "', setting active stage '" << activeStageCached << "'.";
+
+			ConfigPackageUtility::SetActiveStageToFile(package, activeStageCached);
+		}
+	}
+}
+
 void ApiListener::SetActivePackageStage(const String& package, const String& stage)
 {
 	boost::mutex::scoped_lock lock(m_ActivePackageStagesLock);

lib/remote/apilistener.hpp

@@ -124,6 +124,8 @@ private:
 	Timer::Ptr m_ReconnectTimer;
 	Timer::Ptr m_AuthorityTimer;
 	Timer::Ptr m_CleanupCertificateRequestsTimer;
+	Timer::Ptr m_ApiPackageIntegrityTimer;
+
 	Endpoint::Ptr m_LocalEndpoint;
 
 	static ApiListener::Ptr m_Instance;
@@ -131,6 +133,7 @@ private:
 	void ApiTimerHandler();
 	void ApiReconnectTimerHandler();
 	void CleanupCertificateRequestsTimerHandler();
+	void CheckApiPackageIntegrity();
 
 	bool AddListener(const String& node, const String& service);
 	void AddConnection(const Endpoint::Ptr& endpoint);

lib/remote/configpackageutility.cpp

@@ -269,6 +269,17 @@ String ConfigPackageUtility::GetActiveStageFromFile(const String& packageName)
 	return stage.Trim();
 }
 
+void ConfigPackageUtility::SetActiveStageToFile(const String& packageName, const String& stageName)
+{
+	boost::mutex::scoped_lock lock(GetStaticMutex());
+
+	String activeStagePath = GetPackageDir() + "/" + packageName + "/active-stage";
+
+	std::ofstream fpActiveStage(activeStagePath.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); //TODO: fstream exceptions
+	fpActiveStage << stageName;
+	fpActiveStage.close();
+}
+
 String ConfigPackageUtility::GetActiveStage(const String& packageName)
 {
 	ApiListener::Ptr listener = ApiListener::GetInstance();
@@ -307,11 +318,7 @@ void ConfigPackageUtility::SetActiveStage(const String& packageName, const Strin
 	listener->SetActivePackageStage(packageName, stageName);
 
 	/* Also update the marker on disk for restarts. */
-	String activeStagePath = GetPackageDir() + "/" + packageName + "/active-stage";
-
-	std::ofstream fpActiveStage(activeStagePath.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); //TODO: fstream exceptions
-	fpActiveStage << stageName;
-	fpActiveStage.close();
+	SetActiveStageToFile(packageName, stageName);
 }
 
 std::vector<std::pair<String, bool> > ConfigPackageUtility::GetFiles(const String& packageName, const String& stageName)

lib/remote/configpackageutility.hpp

@@ -35,6 +35,7 @@ public:
 	static String GetActiveStageFromFile(const String& packageName);
 	static String GetActiveStage(const String& packageName);
 	static void SetActiveStage(const String& packageName, const String& stageName);
+	static void SetActiveStageToFile(const String& packageName, const String& stageName);
 	static void ActivateStage(const String& packageName, const String& stageName);
 	static void AsyncTryActivateStage(const String& packageName, const String& stageName, bool reload);