Use WaitGroup to wait for or abort HTTP requests

The wait group gets passed to HttpServerConnection, then down to the
HttpHandlers. For those handlers that modify the program state, the
wait group is locked so ApiListener will wait on Stop() for the
request to complete. If the request iterates over config objects,
a further check on the state of the wait group is added to abort early
and not delay program shutdown. In that case, 503 responses will be
sent to the client.

Additionally, in HttpServerConnection, no further requests than the
one already started will be allowed once the wait group is joining.

lib/remote/actionshandler.cpp

@@ -16,6 +16,7 @@ thread_local ApiUser::Ptr ActionsHandler::AuthenticatedApiUser;
 REGISTER_URLHANDLER("/v1/actions", ActionsHandler);
 
 bool ActionsHandler::HandleRequest(
+	const WaitGroup::Ptr& waitGroup,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -88,7 +89,28 @@ bool ActionsHandler::HandleRequest(
 	if (params)
 		verbose = HttpUtility::GetLastParameter(params, "verbose");
 
+	std::shared_lock wgLock{*waitGroup, std::try_to_lock};
+	if (!wgLock) {
+		HttpUtility::SendJsonError(response, params, 503, "Shutting down.");
+		return true;
+	}
+
 	for (ConfigObject::Ptr obj : objs) {
+		if (!waitGroup->IsLockable()) {
+			if (wgLock) {
+				wgLock.unlock();
+			}
+
+			results.emplace_back(new Dictionary({
+				{ "type", obj->GetReflectionType()->GetName() },
+				{ "name", obj->GetName() },
+				{ "code", 503 },
+				{ "status", "Action skipped: Shutting down."}
+			}));
+
+			continue;
+		}
+
 		try {
 			results.emplace_back(action->Invoke(obj, params));
 		} catch (const std::exception& ex) {

lib/remote/actionshandler.hpp

@@ -16,6 +16,7 @@ public:
 	static thread_local ApiUser::Ptr AuthenticatedApiUser;
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/apilistener.cpp

@@ -917,7 +917,7 @@ void ApiListener::NewClientHandlerInternal(
 	} else {
 		Log(LogNotice, "ApiListener", "New HTTP client");
 
-		HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, client);
+		HttpServerConnection::Ptr aclient = new HttpServerConnection(m_WaitGroup, identity, verify_ok, client);
 		AddHttpClient(aclient);
 		aclient->Start();
 		shutdownSslConn.Cancel();

lib/remote/configfileshandler.cpp

@@ -14,6 +14,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/config/files", ConfigFilesHandler);
 
 bool ConfigFilesHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/configfileshandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ConfigFilesHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/configpackageshandler.cpp

@@ -11,6 +11,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/config/packages", ConfigPackagesHandler);
 
 bool ConfigPackagesHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/configpackageshandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ConfigPackagesHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/configstageshandler.cpp

@@ -15,6 +15,7 @@ REGISTER_URLHANDLER("/v1/config/stages", ConfigStagesHandler);
 std::atomic<bool> ConfigStagesHandler::m_RunningPackageUpdates (false);
 
 bool ConfigStagesHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/configstageshandler.hpp

@@ -15,6 +15,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ConfigStagesHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/consolehandler.cpp

@@ -54,6 +54,7 @@ static void EnsureFrameCleanupTimer()
 }
 
 bool ConsoleHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/consolehandler.hpp

@@ -23,6 +23,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ConsoleHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/createobjecthandler.cpp

@@ -16,6 +16,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/objects", CreateObjectHandler);
 
 bool CreateObjectHandler::HandleRequest(
+	const WaitGroup::Ptr& waitGroup,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -102,6 +103,12 @@ bool CreateObjectHandler::HandleRequest(
 		return true;
 	}
 
+	std::shared_lock wgLock{*waitGroup, std::try_to_lock};
+	if (!wgLock) {
+		HttpUtility::SendJsonError(response, params, 503, "Shutting down.");
+		return true;
+	}
+
 	/* Object creation can cause multiple errors and optionally diagnostic information.
 	 * We can't use SendJsonError() here.
 	 */

lib/remote/createobjecthandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(CreateObjectHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/deleteobjecthandler.cpp

@@ -16,6 +16,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/objects", DeleteObjectHandler);
 
 bool DeleteObjectHandler::HandleRequest(
+	const WaitGroup::Ptr& waitGroup,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -78,7 +79,30 @@ bool DeleteObjectHandler::HandleRequest(
 
 	bool success = true;
 
+	std::shared_lock wgLock{*waitGroup, std::try_to_lock};
+	if (!wgLock) {
+		HttpUtility::SendJsonError(response, params, 503, "Shutting down.");
+		return true;
+	}
+
 	for (ConfigObject::Ptr obj : objs) {
+		if (!waitGroup->IsLockable()) {
+			if (wgLock) {
+				wgLock.unlock();
+			}
+
+			results.emplace_back(new Dictionary({
+				{ "type", type->GetName() },
+				{ "name", obj->GetName() },
+				{ "code", 503 },
+				{ "status", "Action skipped: Shutting down."}
+			}));
+
+			success = false;
+
+			continue;
+		}
+
 		int code;
 		String status;
 		Array::Ptr errors = new Array();

lib/remote/deleteobjecthandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(DeleteObjectHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/eventshandler.cpp

@@ -40,6 +40,7 @@ const std::map<String, EventType> l_EventTypes ({
 const String l_ApiQuery ("<API query>");
 
 bool EventsHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/eventshandler.hpp

@@ -15,6 +15,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(EventsHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/httphandler.cpp

@@ -47,6 +47,7 @@ void HttpHandler::Register(const Url::Ptr& url, const HttpHandler::Ptr& handler)
 }
 
 void HttpHandler::ProcessRequest(
+	const WaitGroup::Ptr& waitGroup,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -108,7 +109,7 @@ void HttpHandler::ProcessRequest(
 	 */
 	try {
 		for (const HttpHandler::Ptr& handler : handlers) {
-			if (handler->HandleRequest(stream, user, request, url, response, params, yc, server)) {
+			if (handler->HandleRequest(waitGroup, stream, user, request, url, response, params, yc, server)) {
 				processed = true;
 				break;
 			}

lib/remote/httphandler.hpp

@@ -27,6 +27,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(HttpHandler);
 
 	virtual bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -39,6 +40,7 @@ public:
 
 	static void Register(const Url::Ptr& url, const HttpHandler::Ptr& handler);
 	static void ProcessRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/httpserverconnection.cpp

@@ -35,13 +35,13 @@ using namespace icinga;
 
 auto const l_ServerHeader ("Icinga/" + Application::GetAppVersion());
 
-HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream)
-	: HttpServerConnection(identity, authenticated, stream, IoEngine::Get().GetIoContext())
+HttpServerConnection::HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream)
+	: HttpServerConnection(waitGroup, identity, authenticated, stream, IoEngine::Get().GetIoContext())
 {
 }
 
-HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream, boost::asio::io_context& io)
-	: m_Stream(stream), m_Seen(Utility::GetTime()), m_IoStrand(io), m_ShuttingDown(false), m_HasStartedStreaming(false),
+HttpServerConnection::HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream, boost::asio::io_context& io)
+	: m_WaitGroup(waitGroup), m_Stream(stream), m_Seen(Utility::GetTime()), m_IoStrand(io), m_ShuttingDown(false), m_HasStartedStreaming(false),
 	m_CheckLivenessTimer(io)
 {
 	if (authenticated) {
@@ -419,6 +419,7 @@ bool ProcessRequest(
 	boost::beast::http::response<boost::beast::http::string_body>& response,
 	HttpServerConnection& server,
 	bool& hasStartedStreaming,
+	const WaitGroup::Ptr& waitGroup,
 	std::chrono::steady_clock::duration& cpuBoundWorkTime,
 	boost::asio::yield_context& yc
 )
@@ -431,7 +432,7 @@ bool ProcessRequest(
 		CpuBoundWork handlingRequest (yc);
 		cpuBoundWorkTime = std::chrono::steady_clock::now() - start;
 
-		HttpHandler::ProcessRequest(stream, authenticatedUser, request, response, yc, server);
+		HttpHandler::ProcessRequest(waitGroup, stream, authenticatedUser, request, response, yc, server);
 	} catch (const std::exception& ex) {
 		if (hasStartedStreaming) {
 			return false;
@@ -477,7 +478,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc)
 		 */
 		beast::flat_buffer buf;
 
-		for (;;) {
+		while (m_WaitGroup->IsLockable()) {
 			m_Seen = Utility::GetTime();
 
 			http::parser<true, http::string_body> parser;
@@ -548,7 +549,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc)
 
 			m_Seen = std::numeric_limits<decltype(m_Seen)>::max();
 
-			if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, *this, m_HasStartedStreaming, cpuBoundWorkTime, yc)) {
+			if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, *this, m_HasStartedStreaming, m_WaitGroup, cpuBoundWorkTime, yc)) {
 				break;
 			}
 

lib/remote/httpserverconnection.hpp

@@ -6,6 +6,7 @@
 #include "remote/apiuser.hpp"
 #include "base/string.hpp"
 #include "base/tlsstream.hpp"
+#include "base/wait-group.hpp"
 #include <memory>
 #include <boost/asio/deadline_timer.hpp>
 #include <boost/asio/io_context.hpp>
@@ -25,14 +26,15 @@ class HttpServerConnection final : public Object
 public:
 	DECLARE_PTR_TYPEDEFS(HttpServerConnection);
 
-	HttpServerConnection(const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream);
+	HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated,
+		const Shared<AsioTlsStream>::Ptr& stream);
 
 	void Start();
 	void StartStreaming();
-
 	bool Disconnected();
 
 private:
+	WaitGroup::Ptr m_WaitGroup;
 	ApiUser::Ptr m_ApiUser;
 	Shared<AsioTlsStream>::Ptr m_Stream;
 	double m_Seen;
@@ -42,7 +44,8 @@ private:
 	bool m_HasStartedStreaming;
 	boost::asio::deadline_timer m_CheckLivenessTimer;
 
-	HttpServerConnection(const String& identity, bool authenticated, const Shared<AsioTlsStream>::Ptr& stream, boost::asio::io_context& io);
+	HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated,
+		const Shared<AsioTlsStream>::Ptr& stream, boost::asio::io_context& io);
 
 	void Disconnect(boost::asio::yield_context yc);
 

lib/remote/infohandler.cpp

@@ -9,6 +9,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/", InfoHandler);
 
 bool InfoHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/infohandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(InfoHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/mallocinfohandler.cpp

@@ -18,6 +18,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/debug/malloc_info", MallocInfoHandler);
 
 bool MallocInfoHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream&,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/mallocinfohandler.hpp

@@ -13,6 +13,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(MallocInfoHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/modifyobjecthandler.cpp

@@ -14,6 +14,7 @@ using namespace icinga;
 REGISTER_URLHANDLER("/v1/objects", ModifyObjectHandler);
 
 bool ModifyObjectHandler::HandleRequest(
+	const WaitGroup::Ptr& waitGroup,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,
@@ -104,12 +105,31 @@ bool ModifyObjectHandler::HandleRequest(
 
 	ArrayData results;
 
+	std::shared_lock wgLock{*waitGroup, std::try_to_lock};
+	if (!wgLock) {
+		HttpUtility::SendJsonError(response, params, 503, "Shutting down.");
+		return true;
+	}
+
 	for (ConfigObject::Ptr obj : objs) {
 		Dictionary::Ptr result1 = new Dictionary();
 
 		result1->Set("type", type->GetName());
 		result1->Set("name", obj->GetName());
 
+		if (!waitGroup->IsLockable()) {
+			if (wgLock) {
+				wgLock.unlock();
+			}
+
+			result1->Set("code", 503);
+			result1->Set("status", "Action skipped: Shutting down.");
+
+			results.emplace_back(std::move(result1));
+
+			continue;
+		}
+
 		String key;
 
 		// Lock the object name of the given type to prevent from being modified/deleted concurrently.

lib/remote/modifyobjecthandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ModifyObjectHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/objectqueryhandler.cpp

@@ -89,6 +89,7 @@ Dictionary::Ptr ObjectQueryHandler::SerializeObjectAttrs(const Object::Ptr& obje
 }
 
 bool ObjectQueryHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/objectqueryhandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(ObjectQueryHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/statushandler.cpp

@@ -69,6 +69,7 @@ public:
 };
 
 bool StatusHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/statushandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(StatusHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/templatequeryhandler.cpp

@@ -76,6 +76,7 @@ public:
 };
 
 bool TemplateQueryHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/templatequeryhandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(TemplateQueryHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/typequeryhandler.cpp

@@ -47,6 +47,7 @@ public:
 };
 
 bool TypeQueryHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/typequeryhandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(TypeQueryHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/variablequeryhandler.cpp

@@ -57,6 +57,7 @@ public:
 };
 
 bool VariableQueryHandler::HandleRequest(
+	const WaitGroup::Ptr&,
 	AsioTlsStream& stream,
 	const ApiUser::Ptr& user,
 	boost::beast::http::request<boost::beast::http::string_body>& request,

lib/remote/variablequeryhandler.hpp

@@ -14,6 +14,7 @@ public:
 	DECLARE_PTR_TYPEDEFS(VariableQueryHandler);
 
 	bool HandleRequest(
+		const WaitGroup::Ptr& waitGroup,
 		AsioTlsStream& stream,
 		const ApiUser::Ptr& user,
 		boost::beast::http::request<boost::beast::http::string_body>& request,