From c262c701d95246ab2a9f2e4911254e3bad46d433 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 7 Feb 2019 16:33:59 +0100 Subject: [PATCH 01/67] Require Boost v1.66.0+ --- CMakeLists.txt | 2 +- doc/21-development.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d94eb79ff..359fa16a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ cmake_minimum_required(VERSION 2.8.8) -set(BOOST_MIN_VERSION "1.53.0") +set(BOOST_MIN_VERSION "1.66.0") project(icinga2) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") diff --git a/doc/21-development.md b/doc/21-development.md index aa50f0097..72bca8f33 100644 --- a/doc/21-development.md +++ b/doc/21-development.md @@ -1095,8 +1095,8 @@ Icinga application using a dist tarball (including notes for distributions): - SUSE: libopenssl-devel (for SLES 11: libopenssl1-devel) - Debian/Ubuntu: libssl-dev - Alpine: libressl-dev -* Boost library and header files >= 1.53.0 - - RHEL/Fedora: boost153-devel +* Boost library and header files >= 1.66.0 + - RHEL/Fedora: boost166-devel - Debian/Ubuntu: libboost-all-dev - Alpine: boost-dev * GNU bison (bison) From f38c68a4c6d5ab590b679361bf7105bf800f048e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Feb 2019 11:51:22 +0100 Subject: [PATCH 02/67] Require Boost context, coroutine and date_time --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 359fa16a2..a3ee2e503 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,7 +132,7 @@ if(LOGROTATE_HAS_SU) set(LOGROTATE_USE_SU "\n\tsu ${ICINGA2_USER} ${ICINGA2_GROUP}") endif() -find_package(Boost ${BOOST_MIN_VERSION} COMPONENTS thread system program_options regex REQUIRED) +find_package(Boost ${BOOST_MIN_VERSION} COMPONENTS context coroutine date_time thread system program_options regex REQUIRED) link_directories(${Boost_LIBRARY_DIRS}) include_directories(${Boost_INCLUDE_DIRS}) From 7c7c5e28f5f4354aa86cd0469d64ce6a3a80ef7e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Feb 2019 11:47:03 +0100 Subject: [PATCH 03/67] Implement LazyInit --- lib/base/CMakeLists.txt | 1 + lib/base/lazy-init.hpp | 89 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 lib/base/lazy-init.hpp diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index 6d7661080..9e04de723 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -35,6 +35,7 @@ set(base_SOURCES function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp initialize.cpp initialize.hpp json.cpp json.hpp json-script.cpp + lazy-init.hpp library.cpp library.hpp loader.cpp loader.hpp logger.cpp logger.hpp logger-ti.hpp diff --git a/lib/base/lazy-init.hpp b/lib/base/lazy-init.hpp new file mode 100644 index 000000000..dd20a80e5 --- /dev/null +++ b/lib/base/lazy-init.hpp @@ -0,0 +1,89 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef LAZY_INIT +#define LAZY_INIT + +#include +#include +#include +#include + +namespace icinga +{ + +/** + * Lazy object initialization abstraction inspired from + * . + * + * @ingroup base + */ +template +class LazyInit +{ +public: + inline + LazyInit(std::function initializer = []() { return T(); }) : m_Initializer(std::move(initializer)) + { + m_Underlying.store(nullptr, std::memory_order_release); + } + + LazyInit(const LazyInit&) = delete; + LazyInit(LazyInit&&) = delete; + LazyInit& operator=(const LazyInit&) = delete; + LazyInit& operator=(LazyInit&&) = delete; + + inline + ~LazyInit() + { + auto ptr (m_Underlying.load(std::memory_order_acquire)); + + if (ptr != nullptr) { + delete ptr; + } + } + + inline + T& Get() + { + auto ptr (m_Underlying.load(std::memory_order_acquire)); + + if (ptr == nullptr) { + std::unique_lock lock (m_Mutex); + + ptr = m_Underlying.load(std::memory_order_acquire); + + if (ptr == nullptr) { + ptr = new T(m_Initializer()); + m_Underlying.store(ptr, std::memory_order_release); + } + } + + return *ptr; + } + +private: + std::function m_Initializer; + std::mutex m_Mutex; + std::atomic m_Underlying; +}; + +} + +#endif /* LAZY_INIT */ From c547e9a86398ca524625c35d3815fb476f8f1311 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Feb 2019 10:05:24 +0100 Subject: [PATCH 04/67] Implement basic I/O engine --- lib/base/CMakeLists.txt | 1 + lib/base/io-engine.cpp | 96 +++++++++++++++++++++++++++++++++++++++++ lib/base/io-engine.hpp | 85 ++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 lib/base/io-engine.cpp create mode 100644 lib/base/io-engine.hpp diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index 9e04de723..9260b31ed 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -34,6 +34,7 @@ set(base_SOURCES filelogger.cpp filelogger.hpp filelogger-ti.hpp function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp initialize.cpp initialize.hpp + io-engine.cpp io-engine.hpp json.cpp json.hpp json-script.cpp lazy-init.hpp library.cpp library.hpp diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp new file mode 100644 index 000000000..e1aeeb094 --- /dev/null +++ b/lib/base/io-engine.cpp @@ -0,0 +1,96 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "base/exception.hpp" +#include "base/io-engine.hpp" +#include "base/lazy-init.hpp" +#include "base/logger.hpp" +#include +#include +#include +#include +#include +#include + +using namespace icinga; + +CpuBoundWork::CpuBoundWork(boost::asio::yield_context yc) +{ + auto& ioEngine (IoEngine::Get()); + + for (;;) { + auto availableSlots (ioEngine.m_CpuBoundSemaphore.fetch_sub(1)); + + if (availableSlots < 1) { + ioEngine.m_CpuBoundSemaphore.fetch_add(1); + ioEngine.m_AlreadyExpiredTimer.async_wait(yc); + continue; + } + + break; + } +} + +CpuBoundWork::~CpuBoundWork() +{ + IoEngine::Get().m_CpuBoundSemaphore.fetch_add(1); +} + +LazyInit> IoEngine::m_Instance ([]() { return std::unique_ptr(new IoEngine()); }); + +IoEngine& IoEngine::Get() +{ + return *m_Instance.Get(); +} + +boost::asio::io_service& IoEngine::GetIoService() +{ + return m_IoService; +} + +IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_AlreadyExpiredTimer(m_IoService) +{ + m_AlreadyExpiredTimer.expires_at(boost::posix_time::neg_infin); + + auto concurrency (std::thread::hardware_concurrency()); + + if (concurrency < 2) { + m_CpuBoundSemaphore.store(1); + } else { + m_CpuBoundSemaphore.store(concurrency - 1u); + } + + for (auto i (std::thread::hardware_concurrency()); i; --i) { + std::thread(&IoEngine::RunEventLoop, this).detach(); + } +} + +void IoEngine::RunEventLoop() +{ + for (;;) { + try { + m_IoService.run(); + + break; + } catch (const std::exception& e) { + Log(LogCritical, "IoEngine", "Exception during I/O operation!"); + Log(LogDebug, "IoEngine") << "Exception during I/O operation: " << DiagnosticInformation(e); + } + } +} diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp new file mode 100644 index 000000000..df84df9ce --- /dev/null +++ b/lib/base/io-engine.hpp @@ -0,0 +1,85 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef IO_ENGINE_H +#define IO_ENGINE_H + +/** + * Boost.Coroutine2 (the successor of Boost.Coroutine) + * (1) doesn't even exist in old Boost versions and + * (2) isn't supported by ASIO, yet. + */ +#define BOOST_COROUTINES_NO_DEPRECATION_WARNING 1 + +#include "base/lazy-init.hpp" +#include +#include +#include +#include +#include + +/** + * Scope lock for CPU-bound work done in an I/O thread + * + * @ingroup base + */ +class CpuBoundWork +{ +public: + CpuBoundWork(boost::asio::yield_context yc); + CpuBoundWork(const CpuBoundWork&) = delete; + CpuBoundWork(CpuBoundWork&&) = delete; + CpuBoundWork& operator=(const CpuBoundWork&) = delete; + CpuBoundWork& operator=(CpuBoundWork&&) = delete; + ~CpuBoundWork(); +}; + +/** + * Async I/O engine + * + * @ingroup base + */ +class IoEngine +{ + friend CpuBoundWork; + +public: + IoEngine(const IoEngine&) = delete; + IoEngine(IoEngine&&) = delete; + IoEngine& operator=(const IoEngine&) = delete; + IoEngine& operator=(IoEngine&&) = delete; + + static IoEngine& Get(); + + boost::asio::io_service& GetIoService(); + +private: + IoEngine(); + + void RunEventLoop(); + + static LazyInit> m_Instance; + + boost::asio::io_service m_IoService; + boost::asio::io_service::work m_KeepAlive; + boost::asio::deadline_timer m_AlreadyExpiredTimer; + std::atomic_uint_fast32_t m_CpuBoundSemaphore; +}; + +#endif /* IO_ENGINE_H */ From e4f3422b3a69dd0181de531740c7f008d34dd7cb Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Feb 2019 11:43:47 +0100 Subject: [PATCH 05/67] ApiListener: listen(2) via Boost ASIO --- lib/remote/apilistener.cpp | 62 ++++++++++++++++++++------------------ lib/remote/apilistener.hpp | 5 ++- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index a26846d38..d8e1832ee 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -7,6 +7,7 @@ #include "remote/jsonrpc.hpp" #include "remote/apifunction.hpp" #include "base/convert.hpp" +#include "base/io-engine.hpp" #include "base/netstring.hpp" #include "base/json.hpp" #include "base/configtype.hpp" @@ -18,7 +19,12 @@ #include "base/context.hpp" #include "base/statsfunction.hpp" #include "base/exception.hpp" +#include +#include +#include +#include #include +#include using namespace icinga; @@ -326,6 +332,10 @@ bool ApiListener::IsMaster() const */ bool ApiListener::AddListener(const String& node, const String& service) { + namespace asio = boost::asio; + namespace ip = asio::ip; + using ip::tcp; + ObjectLock olock(this); std::shared_ptr sslContext = m_SSLContext; @@ -335,47 +345,40 @@ bool ApiListener::AddListener(const String& node, const String& service) return false; } - TcpSocket::Ptr server = new TcpSocket(); + auto& io (IoEngine::Get().GetIoService()); + auto acceptor (std::make_shared(io)); try { - server->Bind(node, service, AF_UNSPEC); + tcp::resolver resolver (io); + tcp::resolver::query query (node, service, tcp::resolver::query::passive); + auto endpoint (resolver.resolve(query)->endpoint()); + + acceptor->open(endpoint.protocol()); + acceptor->set_option(ip::v6_only(false)); + acceptor->set_option(tcp::acceptor::reuse_address(true)); + acceptor->bind(endpoint); } catch (const std::exception&) { Log(LogCritical, "ApiListener") << "Cannot bind TCP socket for host '" << node << "' on port '" << service << "'."; return false; } + acceptor->listen(INT_MAX); + + auto localEndpoint (acceptor->local_endpoint()); + Log(LogInformation, "ApiListener") - << "Started new listener on '" << server->GetClientAddress() << "'"; + << "Started new listener on '[" << localEndpoint.address() << "]:" << localEndpoint.port() << "'"; - std::thread thread(std::bind(&ApiListener::ListenerThreadProc, this, server)); - thread.detach(); + asio::spawn(io, [acceptor](asio::yield_context yc) { + // TODO + }); - m_Servers.insert(server); - - UpdateStatusFile(server); + UpdateStatusFile(localEndpoint); return true; } -void ApiListener::ListenerThreadProc(const Socket::Ptr& server) -{ - Utility::SetThreadName("API Listener"); - - server->Listen(); - - for (;;) { - try { - Socket::Ptr client = server->Accept(); - - /* Use dynamic thread pool with additional on demand resources with fast throughput. */ - EnqueueAsyncCallback(std::bind(&ApiListener::NewClientHandler, this, client, String(), RoleServer), LowLatencyScheduler); - } catch (const std::exception&) { - Log(LogCritical, "ApiListener", "Cannot accept new connection."); - } - } -} - /** * Creates a new JSON-RPC client and connects to the specified endpoint. * @@ -1513,14 +1516,13 @@ String ApiListener::GetFromZoneName(const Zone::Ptr& fromZone) return fromZoneName; } -void ApiListener::UpdateStatusFile(TcpSocket::Ptr socket) +void ApiListener::UpdateStatusFile(boost::asio::ip::tcp::endpoint localEndpoint) { String path = Configuration::CacheDir + "/api-state.json"; - std::pair details = socket->GetClientAddressDetails(); Utility::SaveJsonFile(path, 0644, new Dictionary({ - {"host", details.first}, - {"port", Convert::ToLong(details.second)} + {"host", String(localEndpoint.address().to_string())}, + {"port", localEndpoint.port()} })); } diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 54b96dee5..96861d74b 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -14,6 +14,7 @@ #include "base/tcpsocket.hpp" #include "base/tlsstream.hpp" #include "base/threadpool.hpp" +#include #include namespace icinga @@ -106,7 +107,6 @@ protected: private: std::shared_ptr m_SSLContext; - std::set m_Servers; mutable boost::mutex m_AnonymousClientsLock; mutable boost::mutex m_HttpClientsLock; @@ -130,7 +130,6 @@ private: void NewClientHandler(const Socket::Ptr& client, const String& hostname, ConnectionRole role); void NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role); - void ListenerThreadProc(const Socket::Ptr& server); static ThreadPool& GetTP(); static void EnqueueAsyncCallback(const std::function& callback, SchedulerPolicy policy = DefaultScheduler); @@ -154,7 +153,7 @@ private: static void CopyCertificateFile(const String& oldCertPath, const String& newCertPath); - void UpdateStatusFile(TcpSocket::Ptr socket); + void UpdateStatusFile(boost::asio::ip::tcp::endpoint localEndpoint); void RemoveStatusFile(); /* filesync */ From 2615967e7f461b502c2394d1f80ff3be0948bc5b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Feb 2019 14:23:10 +0100 Subject: [PATCH 06/67] Make ApiListener#m_SSLContext a Boost ASIO SSL context --- lib/base/tlsstream.cpp | 25 +++++++++++- lib/base/tlsstream.hpp | 4 ++ lib/base/tlsutility.cpp | 83 +++++++++++++++++++++++++------------- lib/base/tlsutility.hpp | 8 ++-- lib/remote/apilistener.cpp | 11 +++-- lib/remote/apilistener.hpp | 3 +- 6 files changed, 97 insertions(+), 37 deletions(-) diff --git a/lib/base/tlsstream.cpp b/lib/base/tlsstream.cpp index 5d7678417..129d0bc74 100644 --- a/lib/base/tlsstream.cpp +++ b/lib/base/tlsstream.cpp @@ -6,6 +6,7 @@ #include "base/logger.hpp" #include "base/configuration.hpp" #include "base/convert.hpp" +#include #include #ifndef _WIN32 @@ -26,6 +27,28 @@ bool TlsStream::m_SSLIndexInitialized = false; * @param sslContext The SSL context for the client. */ TlsStream::TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, const std::shared_ptr& sslContext) + : TlsStream(socket, hostname, role, sslContext.get()) +{ +} + +/** + * Constructor for the TlsStream class. + * + * @param role The role of the client. + * @param sslContext The SSL context for the client. + */ +TlsStream::TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, const std::shared_ptr& sslContext) + : TlsStream(socket, hostname, role, sslContext->native_handle()) +{ +} + +/** + * Constructor for the TlsStream class. + * + * @param role The role of the client. + * @param sslContext The SSL context for the client. + */ +TlsStream::TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, SSL_CTX* sslContext) : SocketEvents(socket), m_Eof(false), m_HandshakeOK(false), m_VerifyOK(true), m_ErrorCode(0), m_ErrorOccurred(false), m_Socket(socket), m_Role(role), m_SendQ(new FIFO()), m_RecvQ(new FIFO()), m_CurrentAction(TlsActionNone), m_Retry(false), m_Shutdown(false) @@ -33,7 +56,7 @@ TlsStream::TlsStream(const Socket::Ptr& socket, const String& hostname, Connecti std::ostringstream msgbuf; char errbuf[120]; - m_SSL = std::shared_ptr(SSL_new(sslContext.get()), SSL_free); + m_SSL = std::shared_ptr(SSL_new(sslContext), SSL_free); if (!m_SSL) { msgbuf << "SSL_new() failed with code " << ERR_peek_error() << ", \"" << ERR_error_string(ERR_peek_error(), errbuf) << "\""; diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index 8af5fb58e..cd3d0abfa 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -9,6 +9,7 @@ #include "base/stream.hpp" #include "base/tlsutility.hpp" #include "base/fifo.hpp" +#include namespace icinga { @@ -32,6 +33,7 @@ public: DECLARE_PTR_TYPEDEFS(TlsStream); TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, const std::shared_ptr& sslContext = MakeSSLContext()); + TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, const std::shared_ptr& sslContext); ~TlsStream() override; Socket::Ptr GetSocket() const; @@ -80,6 +82,8 @@ private: static int m_SSLIndex; static bool m_SSLIndexInitialized; + TlsStream(const Socket::Ptr& socket, const String& hostname, ConnectionRole role, SSL_CTX* sslContext); + void OnEvent(int revents) override; void HandleError() const; diff --git a/lib/base/tlsutility.cpp b/lib/base/tlsutility.cpp index 35f4d3ba5..57f8d1901 100644 --- a/lib/base/tlsutility.cpp +++ b/lib/base/tlsutility.cpp @@ -7,6 +7,7 @@ #include "base/utility.hpp" #include "base/application.hpp" #include "base/exception.hpp" +#include #include namespace icinga @@ -57,35 +58,23 @@ void InitializeOpenSSL() l_SSLInitialized = true; } -/** - * Initializes an SSL context using the specified certificates. - * - * @param pubkey The public key. - * @param privkey The matching private key. - * @param cakey CA certificate chain file. - * @returns An SSL context. - */ -std::shared_ptr MakeSSLContext(const String& pubkey, const String& privkey, const String& cakey) +static void SetupSslContext(SSL_CTX *sslContext, const String& pubkey, const String& privkey, const String& cakey) { char errbuf[120]; - InitializeOpenSSL(); - - std::shared_ptr sslContext = std::shared_ptr(SSL_CTX_new(SSLv23_method()), SSL_CTX_free); - long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_CIPHER_SERVER_PREFERENCE; #ifdef SSL_OP_NO_COMPRESSION flags |= SSL_OP_NO_COMPRESSION; #endif /* SSL_OP_NO_COMPRESSION */ - SSL_CTX_set_options(sslContext.get(), flags); + SSL_CTX_set_options(sslContext, flags); - SSL_CTX_set_mode(sslContext.get(), SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); - SSL_CTX_set_session_id_context(sslContext.get(), (const unsigned char *)"Icinga 2", 8); + SSL_CTX_set_mode(sslContext, SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_CTX_set_session_id_context(sslContext, (const unsigned char *)"Icinga 2", 8); if (!pubkey.IsEmpty()) { - if (!SSL_CTX_use_certificate_chain_file(sslContext.get(), pubkey.CStr())) { + if (!SSL_CTX_use_certificate_chain_file(sslContext, pubkey.CStr())) { Log(LogCritical, "SSL") << "Error with public key file '" << pubkey << "': " << ERR_peek_error() << ", \"" << ERR_error_string(ERR_peek_error(), errbuf) << "\""; BOOST_THROW_EXCEPTION(openssl_error() @@ -96,7 +85,7 @@ std::shared_ptr MakeSSLContext(const String& pubkey, const String& priv } if (!privkey.IsEmpty()) { - if (!SSL_CTX_use_PrivateKey_file(sslContext.get(), privkey.CStr(), SSL_FILETYPE_PEM)) { + if (!SSL_CTX_use_PrivateKey_file(sslContext, privkey.CStr(), SSL_FILETYPE_PEM)) { Log(LogCritical, "SSL") << "Error with private key file '" << privkey << "': " << ERR_peek_error() << ", \"" << ERR_error_string(ERR_peek_error(), errbuf) << "\""; BOOST_THROW_EXCEPTION(openssl_error() @@ -105,7 +94,7 @@ std::shared_ptr MakeSSLContext(const String& pubkey, const String& priv << boost::errinfo_file_name(privkey)); } - if (!SSL_CTX_check_private_key(sslContext.get())) { + if (!SSL_CTX_check_private_key(sslContext)) { Log(LogCritical, "SSL") << "Error checking private key '" << privkey << "': " << ERR_peek_error() << ", \"" << ERR_error_string(ERR_peek_error(), errbuf) << "\""; BOOST_THROW_EXCEPTION(openssl_error() @@ -115,7 +104,7 @@ std::shared_ptr MakeSSLContext(const String& pubkey, const String& priv } if (!cakey.IsEmpty()) { - if (!SSL_CTX_load_verify_locations(sslContext.get(), cakey.CStr(), nullptr)) { + if (!SSL_CTX_load_verify_locations(sslContext, cakey.CStr(), nullptr)) { Log(LogCritical, "SSL") << "Error loading and verifying locations in ca key file '" << cakey << "': " << ERR_peek_error() << ", \"" << ERR_error_string(ERR_peek_error(), errbuf) << "\""; BOOST_THROW_EXCEPTION(openssl_error() @@ -136,22 +125,60 @@ std::shared_ptr MakeSSLContext(const String& pubkey, const String& priv << boost::errinfo_file_name(cakey)); } - SSL_CTX_set_client_CA_list(sslContext.get(), cert_names); + SSL_CTX_set_client_CA_list(sslContext, cert_names); } +} + +/** + * Initializes an SSL context using the specified certificates. + * + * @param pubkey The public key. + * @param privkey The matching private key. + * @param cakey CA certificate chain file. + * @returns An SSL context. + */ +std::shared_ptr MakeSSLContext(const String& pubkey, const String& privkey, const String& cakey) +{ + InitializeOpenSSL(); + + std::shared_ptr sslContext = std::shared_ptr(SSL_CTX_new(SSLv23_method()), SSL_CTX_free); + + SetupSslContext(sslContext.get(), pubkey, privkey, cakey); return sslContext; } +/** + * Initializes an SSL context using the specified certificates. + * + * @param pubkey The public key. + * @param privkey The matching private key. + * @param cakey CA certificate chain file. + * @returns An SSL context. + */ +std::shared_ptr MakeAsioSslContext(const String& pubkey, const String& privkey, const String& cakey) +{ + namespace ssl = boost::asio::ssl; + + InitializeOpenSSL(); + + auto context (std::make_shared(ssl::context::sslv23)); + + SetupSslContext(context->native_handle(), pubkey, privkey, cakey); + + return context; +} + /** * Set the cipher list to the specified SSL context. * @param context The ssl context. * @param cipherList The ciper list. **/ -void SetCipherListToSSLContext(const std::shared_ptr& context, const String& cipherList) +void SetCipherListToSSLContext(const std::shared_ptr& context, const String& cipherList) { char errbuf[256]; - if (SSL_CTX_set_cipher_list(context.get(), cipherList.CStr()) == 0) { + if (SSL_CTX_set_cipher_list(context->native_handle(), cipherList.CStr()) == 0) { Log(LogCritical, "SSL") << "Cipher list '" << cipherList @@ -171,9 +198,9 @@ void SetCipherListToSSLContext(const std::shared_ptr& context, const St * @param context The ssl context. * @param tlsProtocolmin The minimum TLS protocol version. */ -void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, const String& tlsProtocolmin) +void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, const String& tlsProtocolmin) { - long flags = SSL_CTX_get_options(context.get()); + long flags = SSL_CTX_get_options(context->native_handle()); flags |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; @@ -190,7 +217,7 @@ void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, cons if (tlsProtocolmin != SSL_TXT_TLSV1) BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid TLS protocol version specified.")); - SSL_CTX_set_options(context.get(), flags); + SSL_CTX_set_options(context->native_handle(), flags); } /** @@ -199,10 +226,10 @@ void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, cons * @param context The SSL context. * @param crlPath The path to the CRL file. */ -void AddCRLToSSLContext(const std::shared_ptr& context, const String& crlPath) +void AddCRLToSSLContext(const std::shared_ptr& context, const String& crlPath) { char errbuf[120]; - X509_STORE *x509_store = SSL_CTX_get_cert_store(context.get()); + X509_STORE *x509_store = SSL_CTX_get_cert_store(context->native_handle()); X509_LOOKUP *lookup; lookup = X509_STORE_add_lookup(x509_store, X509_LOOKUP_file()); diff --git a/lib/base/tlsutility.hpp b/lib/base/tlsutility.hpp index afe21f2e4..69b10786c 100644 --- a/lib/base/tlsutility.hpp +++ b/lib/base/tlsutility.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace icinga @@ -21,9 +22,10 @@ namespace icinga void InitializeOpenSSL(); std::shared_ptr MakeSSLContext(const String& pubkey = String(), const String& privkey = String(), const String& cakey = String()); -void AddCRLToSSLContext(const std::shared_ptr& context, const String& crlPath); -void SetCipherListToSSLContext(const std::shared_ptr& context, const String& cipherList); -void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, const String& tlsProtocolmin); +std::shared_ptr MakeAsioSslContext(const String& pubkey = String(), const String& privkey = String(), const String& cakey = String()); +void AddCRLToSSLContext(const std::shared_ptr& context, const String& crlPath); +void SetCipherListToSSLContext(const std::shared_ptr& context, const String& cipherList); +void SetTlsProtocolminToSSLContext(const std::shared_ptr& context, const String& tlsProtocolmin); String GetCertificateCN(const std::shared_ptr& certificate); std::shared_ptr GetX509Certificate(const String& pemfile); int MakeX509CSR(const String& cn, const String& keyfile, const String& csrfile = String(), const String& certfile = String(), bool ca = false); diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index d8e1832ee..d680cc76a 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -165,10 +166,12 @@ void ApiListener::OnConfigLoaded() void ApiListener::UpdateSSLContext() { - std::shared_ptr context; + namespace ssl = boost::asio::ssl; + + std::shared_ptr context; try { - context = MakeSSLContext(GetDefaultCertPath(), GetDefaultKeyPath(), GetDefaultCaPath()); + context = MakeAsioSslContext(GetDefaultCertPath(), GetDefaultKeyPath(), GetDefaultCaPath()); } catch (const std::exception&) { BOOST_THROW_EXCEPTION(ScriptError("Cannot make SSL context for cert path: '" + GetDefaultCertPath() + "' key path: '" + GetDefaultKeyPath() + "' ca path: '" + GetDefaultCaPath() + "'.", GetDebugInfo())); @@ -338,7 +341,7 @@ bool ApiListener::AddListener(const String& node, const String& service) ObjectLock olock(this); - std::shared_ptr sslContext = m_SSLContext; + auto sslContext (m_SSLContext); if (!sslContext) { Log(LogCritical, "ApiListener", "SSL context is required for AddListener()"); @@ -389,7 +392,7 @@ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint) { ObjectLock olock(this); - std::shared_ptr sslContext = m_SSLContext; + auto sslContext (m_SSLContext); if (!sslContext) { Log(LogCritical, "ApiListener", "SSL context is required for AddConnection()"); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 96861d74b..1de66ed4f 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -15,6 +15,7 @@ #include "base/tlsstream.hpp" #include "base/threadpool.hpp" #include +#include #include namespace icinga @@ -106,7 +107,7 @@ protected: void ValidateTlsHandshakeTimeout(const Lazy& lvalue, const ValidationUtils& utils) override; private: - std::shared_ptr m_SSLContext; + std::shared_ptr m_SSLContext; mutable boost::mutex m_AnonymousClientsLock; mutable boost::mutex m_HttpClientsLock; From 720c53ab77756ed915c3c1830323f4480498a87d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Feb 2019 18:00:53 +0100 Subject: [PATCH 07/67] ApiListener: perform TLS handshake --- lib/remote/apilistener.cpp | 95 ++++++++++++++++++++++++++++++++++++-- lib/remote/apilistener.hpp | 6 +++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index d680cc76a..578e1a682 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -23,9 +23,13 @@ #include #include #include +#include +#include #include #include #include +#include +#include using namespace icinga; @@ -373,15 +377,36 @@ bool ApiListener::AddListener(const String& node, const String& service) Log(LogInformation, "ApiListener") << "Started new listener on '[" << localEndpoint.address() << "]:" << localEndpoint.port() << "'"; - asio::spawn(io, [acceptor](asio::yield_context yc) { - // TODO - }); + asio::spawn(io, [this, acceptor, sslContext](asio::yield_context yc) { ListenerCoroutineProc(yc, acceptor, sslContext); }); UpdateStatusFile(localEndpoint); return true; } +void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext) +{ + namespace asio = boost::asio; + namespace ssl = asio::ssl; + using asio::ip::tcp; + + auto& io (server->get_io_service()); + auto sslConn (std::make_shared>(io, *sslContext)); + + for (;;) { + try { + server->async_accept(sslConn->lowest_layer(), yc); + } catch (const std::exception& ex) { + Log(LogCritical, "ApiListener") << "Cannot accept new connection: " << DiagnosticInformation(ex, false); + continue; + } + + asio::spawn(io, [this, sslConn](asio::yield_context yc) { NewClientHandler(yc, sslConn, String(), RoleServer); }); + + sslConn = std::make_shared>(io, *sslContext); + } +} + /** * Creates a new JSON-RPC client and connects to the specified endpoint. * @@ -601,6 +626,70 @@ void ApiListener::NewClientHandlerInternal(const Socket::Ptr& client, const Stri } } +void ApiListener::NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role) +{ + try { + NewClientHandlerInternal(yc, client, hostname, role); + } catch (const std::exception& ex) { + Log(LogCritical, "ApiListener") + << "Exception while handling new API client connection: " << DiagnosticInformation(ex, false); + + Log(LogDebug, "ApiListener") + << "Exception while handling new API client connection: " << DiagnosticInformation(ex); + } +} + +/** + * Processes a new client connection. + * + * @param client The new client. + */ +void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role) +{ + namespace ssl = boost::asio::ssl; + + String conninfo; + + { + std::ostringstream conninfo_; + + if (role == RoleClient) { + conninfo_ << "to"; + } else { + conninfo_ << "from"; + } + + auto endpoint (client->lowest_layer().remote_endpoint()); + + conninfo_ << " [" << endpoint.address() << "]:" << endpoint.port(); + + conninfo = conninfo_.str(); + } + + client->set_verify_mode(ssl::verify_peer | ssl::verify_client_once); + + if (role == RoleClient) { + String environmentName = Application::GetAppEnvironment(); + String serverName = hostname; + + if (!environmentName.IsEmpty()) + serverName += ":" + environmentName; + +#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME + if (!hostname.IsEmpty()) { + SSL_set_tlsext_host_name(client->native_handle(), serverName.CStr()); + } +#endif /* SSL_CTRL_SET_TLSEXT_HOSTNAME */ + } + + try { + client->async_handshake(role == RoleClient ? client->client : client->server, yc); + } catch (const std::exception& ex) { + Log(LogCritical, "ApiListener") + << "Client TLS handshake failed (" << conninfo << "): " << DiagnosticInformation(ex, false); + } +} + void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync) { Zone::Ptr eZone = endpoint->GetZone(); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 1de66ed4f..e8b578aba 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -15,7 +15,9 @@ #include "base/tlsstream.hpp" #include "base/threadpool.hpp" #include +#include #include +#include #include namespace icinga @@ -132,6 +134,10 @@ private: void NewClientHandler(const Socket::Ptr& client, const String& hostname, ConnectionRole role); void NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role); + void NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role); + void NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role); + void ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext); + static ThreadPool& GetTP(); static void EnqueueAsyncCallback(const std::function& callback, SchedulerPolicy policy = DefaultScheduler); From 539855bac15485e3b00a80b35b43a0f310580076 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Feb 2019 14:56:25 +0100 Subject: [PATCH 08/67] ApiListener: verify peer --- lib/remote/apilistener.cpp | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 578e1a682..4adb9035b 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -24,11 +24,14 @@ #include #include #include +#include #include #include #include #include +#include #include +#include #include using namespace icinga; @@ -668,6 +671,23 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const client->set_verify_mode(ssl::verify_peer | ssl::verify_client_once); + bool verify_ok = false; + String verifyError; + + client->set_verify_callback([&verify_ok, &verifyError](bool preverified, ssl::verify_context& ctx) { + verify_ok = preverified; + + if (!preverified) { + std::ostringstream msgbuf; + int err = X509_STORE_CTX_get_error(ctx.native_handle()); + + msgbuf << "code " << err << ": " << X509_verify_cert_error_string(err); + verifyError = msgbuf.str(); + } + + return preverified; + }); + if (role == RoleClient) { String environmentName = Application::GetAppEnvironment(); String serverName = hostname; @@ -687,6 +707,51 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const } catch (const std::exception& ex) { Log(LogCritical, "ApiListener") << "Client TLS handshake failed (" << conninfo << "): " << DiagnosticInformation(ex, false); + return; + } + + std::shared_ptr cert (SSL_get_peer_certificate(client->native_handle()), X509_free); + String identity; + Endpoint::Ptr endpoint; + + if (cert) { + try { + identity = GetCertificateCN(cert); + } catch (const std::exception&) { + Log(LogCritical, "ApiListener") + << "Cannot get certificate common name from cert path: '" << GetDefaultCertPath() << "'."; + return; + } + + if (!hostname.IsEmpty()) { + if (identity != hostname) { + Log(LogWarning, "ApiListener") + << "Unexpected certificate common name while connecting to endpoint '" + << hostname << "': got '" << identity << "'"; + return; + } else if (!verify_ok) { + Log(LogWarning, "ApiListener") + << "Certificate validation failed for endpoint '" << hostname + << "': " << verifyError; + } + } + + if (verify_ok) { + endpoint = Endpoint::GetByName(identity); + } + + Log log(LogInformation, "ApiListener"); + + log << "New client connection for identity '" << identity << "' " << conninfo; + + if (!verify_ok) { + log << " (certificate validation failed: " << verifyError << ")"; + } else if (!endpoint) { + log << " (no Endpoint object found for identity)"; + } + } else { + Log(LogInformation, "ApiListener") + << "New client connection " << conninfo << " (no client certificate)"; } } From e21956e26e7edc20d0b75c331f79405cd980867e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Feb 2019 14:56:47 +0100 Subject: [PATCH 09/67] ApiListener: detect protocol --- lib/base/tlsstream.hpp | 32 +++++++++++++++++++ lib/remote/apilistener.cpp | 63 ++++++++++++++++++++++++++++++-------- lib/remote/apilistener.hpp | 5 ++- 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index cd3d0abfa..cca8cb286 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -9,7 +9,12 @@ #include "base/stream.hpp" #include "base/tlsutility.hpp" #include "base/fifo.hpp" +#include +#include +#include +#include #include +#include namespace icinga { @@ -94,6 +99,33 @@ private: void CloseInternal(bool inDestructor); }; +class AsioTlsStreamHack : public boost::asio::ssl::stream +{ +public: + inline + AsioTlsStreamHack(std::pair& init) + : stream(*init.first, *init.second) + { + } +}; + +class AsioTlsStream : public boost::asio::buffered_stream +{ +public: + inline + AsioTlsStream(boost::asio::io_service& ioService, boost::asio::ssl::context& sslContext) + : AsioTlsStream(std::make_pair(&ioService, &sslContext)) + { + } + +private: + inline + AsioTlsStream(std::pair init) + : buffered_stream(init) + { + } +}; + } #endif /* TLSSTREAM_H */ diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 4adb9035b..3f882a3d1 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -19,13 +19,14 @@ #include "base/context.hpp" #include "base/statsfunction.hpp" #include "base/exception.hpp" +#include #include #include #include #include -#include #include #include +#include #include #include #include @@ -390,11 +391,9 @@ bool ApiListener::AddListener(const String& node, const String& service) void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext) { namespace asio = boost::asio; - namespace ssl = asio::ssl; - using asio::ip::tcp; auto& io (server->get_io_service()); - auto sslConn (std::make_shared>(io, *sslContext)); + auto sslConn (std::make_shared(io, *sslContext)); for (;;) { try { @@ -406,7 +405,7 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std asio::spawn(io, [this, sslConn](asio::yield_context yc) { NewClientHandler(yc, sslConn, String(), RoleServer); }); - sslConn = std::make_shared>(io, *sslContext); + sslConn = std::make_shared(io, *sslContext); } } @@ -629,7 +628,7 @@ void ApiListener::NewClientHandlerInternal(const Socket::Ptr& client, const Stri } } -void ApiListener::NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role) +void ApiListener::NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role) { try { NewClientHandlerInternal(yc, client, hostname, role); @@ -647,9 +646,10 @@ void ApiListener::NewClientHandler(boost::asio::yield_context yc, const std::sha * * @param client The new client. */ -void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role) +void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role) { - namespace ssl = boost::asio::ssl; + namespace asio = boost::asio; + namespace ssl = asio::ssl; String conninfo; @@ -669,12 +669,14 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const conninfo = conninfo_.str(); } - client->set_verify_mode(ssl::verify_peer | ssl::verify_client_once); + auto& sslConn (client->next_layer()); + + sslConn.set_verify_mode(ssl::verify_peer | ssl::verify_client_once); bool verify_ok = false; String verifyError; - client->set_verify_callback([&verify_ok, &verifyError](bool preverified, ssl::verify_context& ctx) { + sslConn.set_verify_callback([&verify_ok, &verifyError](bool preverified, ssl::verify_context& ctx) { verify_ok = preverified; if (!preverified) { @@ -697,20 +699,20 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME if (!hostname.IsEmpty()) { - SSL_set_tlsext_host_name(client->native_handle(), serverName.CStr()); + SSL_set_tlsext_host_name(sslConn.native_handle(), serverName.CStr()); } #endif /* SSL_CTRL_SET_TLSEXT_HOSTNAME */ } try { - client->async_handshake(role == RoleClient ? client->client : client->server, yc); + sslConn.async_handshake(role == RoleClient ? sslConn.client : sslConn.server, yc); } catch (const std::exception& ex) { Log(LogCritical, "ApiListener") << "Client TLS handshake failed (" << conninfo << "): " << DiagnosticInformation(ex, false); return; } - std::shared_ptr cert (SSL_get_peer_certificate(client->native_handle()), X509_free); + std::shared_ptr cert (SSL_get_peer_certificate(sslConn.native_handle()), X509_free); String identity; Endpoint::Ptr endpoint; @@ -753,6 +755,41 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const Log(LogInformation, "ApiListener") << "New client connection " << conninfo << " (no client certificate)"; } + + ClientType ctype; + + if (role != RoleClient) { + { + boost::system::error_code ec; + + if (client->async_fill(yc[ec]) == 0u) { + if (identity.IsEmpty()) { + Log(LogInformation, "ApiListener") + << "No data received on new API connection. " + << "Ensure that the remote endpoints are properly configured in a cluster setup."; + } else { + Log(LogWarning, "ApiListener") + << "No data received on new API connection for identity '" << identity << "'. " + << "Ensure that the remote endpoints are properly configured in a cluster setup."; + } + + return; + } + } + + char firstByte = 0; + + { + asio::mutable_buffer firstByteBuf (&firstByte, 1); + client->peek(firstByteBuf); + } + + if (firstByte >= '0' && firstByte <= '9') { + ctype = ClientJsonRpc; + } else { + ctype = ClientHttp; + } + } } void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync) diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index e8b578aba..093837493 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace icinga @@ -134,8 +133,8 @@ private: void NewClientHandler(const Socket::Ptr& client, const String& hostname, ConnectionRole role); void NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role); - void NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role); - void NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr>& client, const String& hostname, ConnectionRole role); + void NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role); + void NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role); void ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext); static ThreadPool& GetTP(); From 2d7714802d661e02e3c8ac1f7410cc7ffe63780a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 14 Feb 2019 13:10:04 +0100 Subject: [PATCH 10/67] Allow CpuBoundWork to be done before end of scope --- lib/base/io-engine.cpp | 14 +++++++++++++- lib/base/io-engine.hpp | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index e1aeeb094..482d57176 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -31,6 +31,7 @@ using namespace icinga; CpuBoundWork::CpuBoundWork(boost::asio::yield_context yc) + : m_Done(false) { auto& ioEngine (IoEngine::Get()); @@ -49,7 +50,18 @@ CpuBoundWork::CpuBoundWork(boost::asio::yield_context yc) CpuBoundWork::~CpuBoundWork() { - IoEngine::Get().m_CpuBoundSemaphore.fetch_add(1); + if (!m_Done) { + IoEngine::Get().m_CpuBoundSemaphore.fetch_add(1); + } +} + +void CpuBoundWork::Done() +{ + if (!m_Done) { + IoEngine::Get().m_CpuBoundSemaphore.fetch_add(1); + + m_Done = true; + } } LazyInit> IoEngine::m_Instance ([]() { return std::unique_ptr(new IoEngine()); }); diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index df84df9ce..efeb56f99 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -48,6 +48,11 @@ public: CpuBoundWork& operator=(const CpuBoundWork&) = delete; CpuBoundWork& operator=(CpuBoundWork&&) = delete; ~CpuBoundWork(); + + void Done(); + +private: + bool m_Done; }; /** From fc22cbaf096d6031abe9fab2fee62c30b521a9e9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 14 Feb 2019 13:12:36 +0100 Subject: [PATCH 11/67] Add HttpUtility::SendJsonBody() overload for Boost/Beast --- lib/remote/httputility.cpp | 10 ++++++++++ lib/remote/httputility.hpp | 2 ++ 2 files changed, 12 insertions(+) diff --git a/lib/remote/httputility.cpp b/lib/remote/httputility.cpp index 043f3cf9e..f7fef7713 100644 --- a/lib/remote/httputility.cpp +++ b/lib/remote/httputility.cpp @@ -5,6 +5,7 @@ #include "base/logger.hpp" #include #include +#include using namespace icinga; @@ -55,6 +56,15 @@ void HttpUtility::SendJsonBody(HttpResponse& response, const Dictionary::Ptr& pa response.WriteBody(body.CStr(), body.GetLength()); } +void HttpUtility::SendJsonBody(boost::beast::http::response& response, const Dictionary::Ptr& params, const Value& val) +{ + namespace http = boost::beast::http; + + response.set(http::field::content_type, "application/json"); + response.body() = JsonEncode(val, params && GetLastParameter(params, "pretty")); + response.set(http::field::content_length, response.body().size()); +} + Value HttpUtility::GetLastParameter(const Dictionary::Ptr& params, const String& key) { Value varr = params->Get(key); diff --git a/lib/remote/httputility.hpp b/lib/remote/httputility.hpp index 0a0418161..e758b2663 100644 --- a/lib/remote/httputility.hpp +++ b/lib/remote/httputility.hpp @@ -6,6 +6,7 @@ #include "remote/httprequest.hpp" #include "remote/httpresponse.hpp" #include "base/dictionary.hpp" +#include namespace icinga { @@ -21,6 +22,7 @@ class HttpUtility public: static Dictionary::Ptr FetchRequestParameters(HttpRequest& request); static void SendJsonBody(HttpResponse& response, const Dictionary::Ptr& params, const Value& val); + static void SendJsonBody(boost::beast::http::response& response, const Dictionary::Ptr& params, const Value& val); static Value GetLastParameter(const Dictionary::Ptr& params, const String& key); static void SendJsonError(HttpResponse& response, const Dictionary::Ptr& params, const int code, const String& verbose = String(), const String& diagnosticInformation = String()); From 04a9879acc0cf33538058725f71a295fd7d43ef5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 14 Feb 2019 16:07:06 +0100 Subject: [PATCH 12/67] Add HttpUtility::SendJsonError() overload for Boost/Beast --- lib/remote/httputility.cpp | 18 ++++++++++++++++++ lib/remote/httputility.hpp | 2 ++ 2 files changed, 20 insertions(+) diff --git a/lib/remote/httputility.cpp b/lib/remote/httputility.cpp index f7fef7713..fa54d9c89 100644 --- a/lib/remote/httputility.cpp +++ b/lib/remote/httputility.cpp @@ -103,6 +103,24 @@ void HttpUtility::SendJsonError(HttpResponse& response, const Dictionary::Ptr& p HttpUtility::SendJsonBody(response, params, result); } +void HttpUtility::SendJsonError(boost::beast::http::response& response, + const Dictionary::Ptr& params, int code, const String& info, const String& diagnosticInformation) +{ + Dictionary::Ptr result = new Dictionary({ { "error", code } }); + + if (!info.IsEmpty()) { + result->Set("status", info); + } + + if (params && HttpUtility::GetLastParameter(params, "verbose") && !diagnosticInformation.IsEmpty()) { + result->Set("diagnostic_information", diagnosticInformation); + } + + response.result(code); + + HttpUtility::SendJsonBody(response, params, result); +} + String HttpUtility::GetErrorNameByCode(const int code) { switch(code) { diff --git a/lib/remote/httputility.hpp b/lib/remote/httputility.hpp index e758b2663..7122e3e71 100644 --- a/lib/remote/httputility.hpp +++ b/lib/remote/httputility.hpp @@ -26,6 +26,8 @@ public: static Value GetLastParameter(const Dictionary::Ptr& params, const String& key); static void SendJsonError(HttpResponse& response, const Dictionary::Ptr& params, const int code, const String& verbose = String(), const String& diagnosticInformation = String()); + static void SendJsonError(boost::beast::http::response& response, const Dictionary::Ptr& params, const int code, + const String& verbose = String(), const String& diagnosticInformation = String()); private: static String GetErrorNameByCode(int code); From 7fe0431ada4c0cd26cde735a3da592d05070c4de Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 14 Feb 2019 16:10:41 +0100 Subject: [PATCH 13/67] HttpServerConnection: verify requests via Boost ASIO + Beast --- lib/remote/apilistener.cpp | 14 +- lib/remote/httpserverconnection.cpp | 691 +++++++++++++++------------- lib/remote/httpserverconnection.hpp | 36 +- 3 files changed, 381 insertions(+), 360 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 3f882a3d1..339ab83bf 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -619,12 +619,6 @@ void ApiListener::NewClientHandlerInternal(const Socket::Ptr& client, const Stri aclient->Disconnect(); } } - } else { - Log(LogNotice, "ApiListener", "New HTTP client"); - - HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, tlsStream); - aclient->Start(); - AddHttpClient(aclient); } } @@ -790,6 +784,14 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const ctype = ClientHttp; } } + + if (ctype != ClientJsonRpc) { + Log(LogNotice, "ApiListener", "New HTTP client"); + + HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, client); + AddHttpClient(aclient); + aclient->Start(); + } } void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 8192dfaca..707c1a635 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -6,306 +6,94 @@ #include "remote/apilistener.hpp" #include "remote/apifunction.hpp" #include "remote/jsonrpc.hpp" +#include "base/application.hpp" #include "base/base64.hpp" #include "base/convert.hpp" #include "base/configtype.hpp" +#include "base/defer.hpp" #include "base/exception.hpp" +#include "base/io-engine.hpp" #include "base/logger.hpp" #include "base/objectlock.hpp" #include "base/timer.hpp" +#include "base/tlsstream.hpp" #include "base/utility.hpp" +#include +#include +#include +#include +#include +#include #include using namespace icinga; -static boost::once_flag l_HttpServerConnectionOnceFlag = BOOST_ONCE_INIT; -static Timer::Ptr l_HttpServerConnectionTimeoutTimer; +auto const l_ServerHeader ("Icinga/" + Application::GetAppVersion()); -HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const TlsStream::Ptr& stream) - : m_Stream(stream), m_Seen(Utility::GetTime()), m_CurrentRequest(stream), m_PendingRequests(0) +HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const std::shared_ptr& stream) + : m_Stream(stream) { - boost::call_once(l_HttpServerConnectionOnceFlag, &HttpServerConnection::StaticInitialize); - - m_RequestQueue.SetName("HttpServerConnection"); - - if (authenticated) + if (authenticated) { m_ApiUser = ApiUser::GetByClientCN(identity); - - /* Cache the peer address. */ - m_PeerAddress = ""; - - if (stream) { - Socket::Ptr socket = m_Stream->GetSocket(); - - if (socket) { - m_PeerAddress = socket->GetPeerAddress(); - } } -} -void HttpServerConnection::StaticInitialize() -{ - l_HttpServerConnectionTimeoutTimer = new Timer(); - l_HttpServerConnectionTimeoutTimer->OnTimerExpired.connect(std::bind(&HttpServerConnection::TimeoutTimerHandler)); - l_HttpServerConnectionTimeoutTimer->SetInterval(5); - l_HttpServerConnectionTimeoutTimer->Start(); + { + std::ostringstream address; + auto endpoint (stream->lowest_layer().remote_endpoint()); + + address << '[' << endpoint.address() << "]:" << endpoint.port(); + + m_PeerAddress = address.str(); + } } void HttpServerConnection::Start() { - /* the stream holds an owning reference to this object through the callback we're registering here */ - m_Stream->RegisterDataHandler(std::bind(&HttpServerConnection::DataAvailableHandler, HttpServerConnection::Ptr(this))); - if (m_Stream->IsDataAvailable()) - DataAvailableHandler(); + namespace asio = boost::asio; + + asio::spawn(IoEngine::Get().GetIoService(), [this](asio::yield_context yc) { ProcessMessages(yc); }); } -ApiUser::Ptr HttpServerConnection::GetApiUser() const +static inline +bool EnsureValidHeaders( + AsioTlsStream& stream, + boost::beast::flat_buffer& buf, + boost::beast::http::parser& parser, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) { - return m_ApiUser; -} + namespace http = boost::beast::http; -TlsStream::Ptr HttpServerConnection::GetStream() const -{ - return m_Stream; -} - -void HttpServerConnection::Disconnect() -{ - boost::recursive_mutex::scoped_try_lock lock(m_DataHandlerMutex); - if (!lock.owns_lock()) { - Log(LogInformation, "HttpServerConnection", "Unable to disconnect Http client, I/O thread busy"); - return; - } - - Log(LogInformation, "HttpServerConnection") - << "HTTP client disconnected (from " << m_PeerAddress << ")"; - - ApiListener::Ptr listener = ApiListener::GetInstance(); - listener->RemoveHttpClient(this); - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(nullptr); - - m_Stream->Close(); -} - -bool HttpServerConnection::ProcessMessage() -{ - bool res; - HttpResponse response(m_Stream, m_CurrentRequest); - - if (!m_CurrentRequest.CompleteHeaders) { + try { try { - res = m_CurrentRequest.ParseHeaders(m_Context, false); - } catch (const std::invalid_argument& ex) { - response.SetStatus(400, "Bad Request"); - String msg = String("

Bad Request

") + ex.what() + "

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - m_Stream->Shutdown(); - - return false; - } catch (const std::exception& ex) { - response.SetStatus(500, "Internal Server Error"); - String msg = "

Internal Server Error

" + DiagnosticInformation(ex) + "

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - m_Stream->Shutdown(); - - return false; - } - return res; - } - - if (!m_CurrentRequest.CompleteHeaderCheck) { - m_CurrentRequest.CompleteHeaderCheck = true; - if (!ManageHeaders(response)) { - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - m_Stream->Shutdown(); - - return false; - } - } - - if (!m_CurrentRequest.CompleteBody) { - try { - res = m_CurrentRequest.ParseBody(m_Context, false); - } catch (const std::invalid_argument& ex) { - response.SetStatus(400, "Bad Request"); - String msg = String("

Bad Request

") + ex.what() + "

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - m_Stream->Shutdown(); - - return false; - } catch (const std::exception& ex) { - response.SetStatus(500, "Internal Server Error"); - String msg = "

Internal Server Error

" + DiagnosticInformation(ex) + "

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - m_Stream->Shutdown(); - - return false; - } - return res; - } - - m_RequestQueue.Enqueue(std::bind(&HttpServerConnection::ProcessMessageAsync, - HttpServerConnection::Ptr(this), m_CurrentRequest, response, m_AuthenticatedUser)); - - m_Seen = Utility::GetTime(); - m_PendingRequests++; - - m_CurrentRequest.~HttpRequest(); - new (&m_CurrentRequest) HttpRequest(m_Stream); - - return false; -} - -bool HttpServerConnection::ManageHeaders(HttpResponse& response) -{ - if (m_CurrentRequest.Headers->Get("expect") == "100-continue") { - String continueResponse = "HTTP/1.1 100 Continue\r\n\r\n"; - m_Stream->Write(continueResponse.CStr(), continueResponse.GetLength()); - } - - /* client_cn matched. */ - if (m_ApiUser) - m_AuthenticatedUser = m_ApiUser; - else - m_AuthenticatedUser = ApiUser::GetByAuthHeader(m_CurrentRequest.Headers->Get("authorization")); - - String requestUrl = m_CurrentRequest.RequestUrl->Format(); - - Log(LogInformation, "HttpServerConnection") - << "Request: " << m_CurrentRequest.RequestMethod << " " << requestUrl - << " (from " << m_PeerAddress << ")" - << ", user: " << (m_AuthenticatedUser ? m_AuthenticatedUser->GetName() : "") << ")"; - - ApiListener::Ptr listener = ApiListener::GetInstance(); - - if (!listener) - return false; - - Array::Ptr headerAllowOrigin = listener->GetAccessControlAllowOrigin(); - - if (headerAllowOrigin && headerAllowOrigin->GetLength() != 0) { - String origin = m_CurrentRequest.Headers->Get("origin"); - { - ObjectLock olock(headerAllowOrigin); - - for (const String& allowedOrigin : headerAllowOrigin) { - if (allowedOrigin == origin) - response.AddHeader("Access-Control-Allow-Origin", origin); - } + http::async_read_header(stream, buf, parser, yc); + } catch (const boost::system::system_error& ex) { + /** + * Unfortunately there's no way to tell an HTTP protocol error + * from an error on a lower layer: + * + * + */ + throw std::invalid_argument(ex.what()); } - response.AddHeader("Access-Control-Allow-Credentials", "true"); - - String accessControlRequestMethodHeader = m_CurrentRequest.Headers->Get("access-control-request-method"); - - if (m_CurrentRequest.RequestMethod == "OPTIONS" && !accessControlRequestMethodHeader.IsEmpty()) { - response.SetStatus(200, "OK"); - - response.AddHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE"); - response.AddHeader("Access-Control-Allow-Headers", "Authorization, X-HTTP-Method-Override"); - - String msg = "Preflight OK"; - response.WriteBody(msg.CStr(), msg.GetLength()); - - response.Finish(); - return false; + switch (parser.get().version()) { + case 10: + case 11: + break; + default: + throw std::invalid_argument("Unsupported HTTP version"); } - } + } catch (const std::invalid_argument& ex) { + response.result(http::status::bad_request); + response.set(http::field::content_type, "text/html"); + response.body() = String("

Bad Request

") + ex.what() + "

"; + response.set(http::field::content_length, response.body().size()); + response.set(http::field::connection, "close"); - if (m_CurrentRequest.RequestMethod != "GET" && m_CurrentRequest.Headers->Get("accept") != "application/json") { - response.SetStatus(400, "Wrong Accept header"); - response.AddHeader("Content-Type", "text/html"); - String msg = "

Accept header is missing or not set to 'application/json'.

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); - return false; - } - - if (!m_AuthenticatedUser) { - Log(LogWarning, "HttpServerConnection") - << "Unauthorized request: " << m_CurrentRequest.RequestMethod << " " << requestUrl; - - response.SetStatus(401, "Unauthorized"); - response.AddHeader("WWW-Authenticate", "Basic realm=\"Icinga 2\""); - - if (m_CurrentRequest.Headers->Get("accept") == "application/json") { - Dictionary::Ptr result = new Dictionary({ - { "error", 401 }, - { "status", "Unauthorized. Please check your user credentials." } - }); - - HttpUtility::SendJsonBody(response, nullptr, result); - } else { - response.AddHeader("Content-Type", "text/html"); - String msg = "

Unauthorized. Please check your user credentials.

"; - response.WriteBody(msg.CStr(), msg.GetLength()); - } - - response.Finish(); - return false; - } - - static const size_t defaultContentLengthLimit = 1 * 1024 * 1024; - size_t maxSize = defaultContentLengthLimit; - - Array::Ptr permissions = m_AuthenticatedUser->GetPermissions(); - - if (permissions) { - ObjectLock olock(permissions); - - for (const Value& permissionInfo : permissions) { - String permission; - - if (permissionInfo.IsObjectType()) - permission = static_cast(permissionInfo)->Get("permission"); - else - permission = permissionInfo; - - static std::vector> specialContentLengthLimits { - { "config/modify", 512 * 1024 * 1024 } - }; - - for (const auto& limitInfo : specialContentLengthLimits) { - if (limitInfo.second <= maxSize) - continue; - - if (Utility::Match(permission, limitInfo.first)) - maxSize = limitInfo.second; - } - } - } - - size_t contentLength = m_CurrentRequest.Headers->Get("content-length"); - - if (contentLength > maxSize) { - response.SetStatus(400, "Bad Request"); - String msg = String("

Content length exceeded maximum

"); - response.WriteBody(msg.CStr(), msg.GetLength()); - response.Finish(); + http::async_write(stream, response, yc); + stream.async_flush(yc); return false; } @@ -313,64 +101,319 @@ bool HttpServerConnection::ManageHeaders(HttpResponse& response) return true; } -void HttpServerConnection::ProcessMessageAsync(HttpRequest& request, HttpResponse& response, const ApiUser::Ptr& user) +static inline +void HandleExpect100( + AsioTlsStream& stream, + boost::beast::http::request& request, + boost::asio::yield_context& yc +) { - response.RebindRequest(request); + namespace http = boost::beast::http; + + if (request[http::field::expect] == "100-continue") { + http::response response; + + response.result(http::status::continue_); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + } +} + +static inline +bool HandleAccessControl( + AsioTlsStream& stream, + boost::beast::http::request& request, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) +{ + namespace http = boost::beast::http; + + auto listener (ApiListener::GetInstance()); + + if (listener) { + auto headerAllowOrigin (listener->GetAccessControlAllowOrigin()); + + if (headerAllowOrigin) { + CpuBoundWork allowOriginHeader (yc); + + auto allowedOrigins (headerAllowOrigin->ToSet()); + + if (!allowedOrigins.empty()) { + auto& origin (request[http::field::origin]); + + if (allowedOrigins.find(origin.to_string()) != allowedOrigins.end()) { + response.set(http::field::access_control_allow_origin, origin); + } + + allowOriginHeader.Done(); + + response.set(http::field::access_control_allow_credentials, "true"); + + if (request.method() == http::verb::options && !request[http::field::access_control_request_method].empty()) { + response.result(http::status::ok); + response.set(http::field::access_control_allow_methods, "GET, POST, PUT, DELETE"); + response.set(http::field::access_control_allow_headers, "Authorization, X-HTTP-Method-Override"); + response.body() = "Preflight OK"; + response.set(http::field::content_length, response.body().size()); + response.set(http::field::connection, "close"); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + + return false; + } + } + } + } + + return true; +} + +static inline +bool EnsureAcceptHeader( + AsioTlsStream& stream, + boost::beast::http::request& request, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) +{ + namespace http = boost::beast::http; + + if (request.method() == http::verb::get && request[http::field::accept] != "application/json") { + response.result(http::status::bad_request); + response.set(http::field::content_type, "text/html"); + response.body() = "

Accept header is missing or not set to 'application/json'.

"; + response.set(http::field::content_length, response.body().size()); + response.set(http::field::connection, "close"); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + + return false; + } + + return true; +} + +static inline +bool EnsureAuthenticatedUser( + AsioTlsStream& stream, + boost::beast::http::request& request, + ApiUser::Ptr& authenticatedUser, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) +{ + namespace http = boost::beast::http; + + if (!authenticatedUser) { + Log(LogWarning, "HttpServerConnection") + << "Unauthorized request: " << request.method_string() << ' ' << request.target(); + + response.result(http::status::unauthorized); + response.set(http::field::www_authenticate, "Basic realm=\"Icinga 2\""); + response.set(http::field::connection, "close"); + + if (request[http::field::accept] == "application/json") { + HttpUtility::SendJsonBody(response, nullptr, new Dictionary({ + { "error", 401 }, + { "status", "Unauthorized. Please check your user credentials." } + })); + } else { + response.set(http::field::content_type, "text/html"); + response.body() = "

Unauthorized. Please check your user credentials.

"; + response.set(http::field::content_length, response.body().size()); + } + + http::async_write(stream, response, yc); + stream.async_flush(yc); + + return false; + } + + return true; +} + +static inline +bool EnsureValidBody( + AsioTlsStream& stream, + boost::beast::flat_buffer& buf, + boost::beast::http::parser& parser, + ApiUser::Ptr& authenticatedUser, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) +{ + namespace http = boost::beast::http; + + { + size_t maxSize = 1024 * 1024; + Array::Ptr permissions = authenticatedUser->GetPermissions(); + + if (permissions) { + CpuBoundWork evalPermissions (yc); + + ObjectLock olock(permissions); + + for (const Value& permissionInfo : permissions) { + String permission; + + if (permissionInfo.IsObjectType()) { + permission = static_cast(permissionInfo)->Get("permission"); + } else { + permission = permissionInfo; + } + + static std::vector> specialContentLengthLimits { + { "config/modify", 512 * 1024 * 1024 } + }; + + for (const auto& limitInfo : specialContentLengthLimits) { + if (limitInfo.second <= maxSize) { + continue; + } + + if (Utility::Match(permission, limitInfo.first)) { + maxSize = limitInfo.second; + } + } + } + } + + parser.body_limit(maxSize); + } try { - HttpHandler::ProcessRequest(user, request, response); + http::async_read(stream, buf, parser, yc); + } catch (const boost::system::system_error& ex) { + /** + * Unfortunately there's no way to tell an HTTP protocol error + * from an error on a lower layer: + * + * + */ + + response.result(http::status::bad_request); + response.set(http::field::content_type, "text/html"); + response.body() = String("

Bad Request

") + ex.what() + "

"; + response.set(http::field::content_length, response.body().size()); + response.set(http::field::connection, "close"); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + + return false; + } + + return true; +} + +static inline +void ProcessRequest( + AsioTlsStream& stream, + boost::beast::http::request& request, + ApiUser::Ptr& authenticatedUser, + boost::beast::http::response& response, + boost::asio::yield_context& yc +) +{ + namespace http = boost::beast::http; + + HttpUtility::SendJsonError(response, nullptr, 503, "Unhandled exception" , ""); + + http::async_write(stream, response, yc); + stream.async_flush(yc); +} + +void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) +{ + namespace beast = boost::beast; + namespace http = beast::http; + + Defer removeHttpClient ([this, &yc]() { + auto listener (ApiListener::GetInstance()); + + if (listener) { + CpuBoundWork removeHttpClient (yc); + + listener->RemoveHttpClient(this); + } + }); + + Defer shutdown ([this, &yc]() { + try { + m_Stream->next_layer().async_shutdown(yc); + } catch (...) { + // https://stackoverflow.com/questions/130117/throwing-exceptions-out-of-a-destructor + } + }); + + try { + beast::flat_buffer buf; + + for (;;) { + http::parser parser; + http::response response; + + parser.header_limit(1024 * 1024); + + response.set(http::field::server, l_ServerHeader); + + if (!EnsureValidHeaders(*m_Stream, buf, parser, response, yc)) { + break; + } + + auto& request (parser.get()); + + { + auto method (http::string_to_verb(request["X-Http-Method-Override"])); + + if (method != http::verb::unknown) { + request.method(method); + } + } + + HandleExpect100(*m_Stream, request, yc); + + auto authenticatedUser (m_ApiUser); + + if (!authenticatedUser) { + CpuBoundWork fetchingAuthenticatedUser (yc); + + authenticatedUser = ApiUser::GetByAuthHeader(request[http::field::authorization].to_string()); + } + + Log(LogInformation, "HttpServerConnection") + << "Request: " << request.method_string() << ' ' << request.target() + << " (from " << m_PeerAddress + << "), user: " << (authenticatedUser ? authenticatedUser->GetName() : "") << ')'; + + if (!HandleAccessControl(*m_Stream, request, response, yc)) { + break; + } + + if (!EnsureAcceptHeader(*m_Stream, request, response, yc)) { + break; + } + + if (!EnsureAuthenticatedUser(*m_Stream, request, authenticatedUser, response, yc)) { + break; + } + + if (!EnsureValidBody(*m_Stream, buf, parser, authenticatedUser, response, yc)) { + break; + } + + ProcessRequest(*m_Stream, request, authenticatedUser, response, yc); + + if (request.version() != 11 || request[http::field::connection] == "close") { + break; + } + } } catch (const std::exception& ex) { Log(LogCritical, "HttpServerConnection") - << "Unhandled exception while processing Http request: " << DiagnosticInformation(ex); - HttpUtility::SendJsonError(response, nullptr, 503, "Unhandled exception" , DiagnosticInformation(ex)); - } - - response.Finish(); - m_PendingRequests--; -} - -void HttpServerConnection::DataAvailableHandler() -{ - bool close = false; - - if (!m_Stream->IsEof()) { - boost::recursive_mutex::scoped_try_lock lock(m_DataHandlerMutex); - if (!lock.owns_lock()) { - Log(LogNotice, "HttpServerConnection", "Unable to process available data, they're already being processed in another thread"); - return; - } - - try { - while (ProcessMessage()) - ; /* empty loop body */ - } catch (const std::exception& ex) { - Log(LogWarning, "HttpServerConnection") - << "Error while reading Http request: " << DiagnosticInformation(ex); - - close = true; - } - } else - close = true; - - if (close) - Disconnect(); -} - -void HttpServerConnection::CheckLiveness() -{ - if (m_Seen < Utility::GetTime() - 10 && m_PendingRequests == 0 && m_Stream->IsEof()) { - Log(LogInformation, "HttpServerConnection") - << "No messages for Http connection have been received in the last 10 seconds."; - Disconnect(); + << "Unhandled exception while processing HTTP request: " << DiagnosticInformation(ex); } } - -void HttpServerConnection::TimeoutTimerHandler() -{ - ApiListener::Ptr listener = ApiListener::GetInstance(); - - for (const HttpServerConnection::Ptr& client : listener->GetHttpClients()) { - client->CheckLiveness(); - } -} - diff --git a/lib/remote/httpserverconnection.hpp b/lib/remote/httpserverconnection.hpp index e27ba839c..3fdbeef50 100644 --- a/lib/remote/httpserverconnection.hpp +++ b/lib/remote/httpserverconnection.hpp @@ -3,12 +3,11 @@ #ifndef HTTPSERVERCONNECTION_H #define HTTPSERVERCONNECTION_H -#include "remote/httprequest.hpp" -#include "remote/httpresponse.hpp" #include "remote/apiuser.hpp" +#include "base/string.hpp" #include "base/tlsstream.hpp" -#include "base/workqueue.hpp" -#include +#include +#include namespace icinga { @@ -23,39 +22,16 @@ class HttpServerConnection final : public Object public: DECLARE_PTR_TYPEDEFS(HttpServerConnection); - HttpServerConnection(const String& identity, bool authenticated, const TlsStream::Ptr& stream); + HttpServerConnection(const String& identity, bool authenticated, const std::shared_ptr& stream); void Start(); - ApiUser::Ptr GetApiUser() const; - bool IsAuthenticated() const; - TlsStream::Ptr GetStream() const; - - void Disconnect(); - private: ApiUser::Ptr m_ApiUser; - ApiUser::Ptr m_AuthenticatedUser; - TlsStream::Ptr m_Stream; - double m_Seen; - HttpRequest m_CurrentRequest; - boost::recursive_mutex m_DataHandlerMutex; - WorkQueue m_RequestQueue; - int m_PendingRequests; + std::shared_ptr m_Stream; String m_PeerAddress; - StreamReadContext m_Context; - - bool ProcessMessage(); - void DataAvailableHandler(); - - static void StaticInitialize(); - static void TimeoutTimerHandler(); - void CheckLiveness(); - - bool ManageHeaders(HttpResponse& response); - - void ProcessMessageAsync(HttpRequest& request, HttpResponse& response, const ApiUser::Ptr&); + void ProcessMessages(boost::asio::yield_context yc); }; } From 9ae1d732af3052865c8bd2b4d1bafd90df0ee682 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 14 Feb 2019 17:27:17 +0100 Subject: [PATCH 14/67] HttpServerConnection: actually handle requests --- lib/remote/httphandler.cpp | 20 +++++++++++++------- lib/remote/httphandler.hpp | 16 ++++++++++++++-- lib/remote/httpserverconnection.cpp | 15 ++++++++++++++- lib/remote/httputility.cpp | 17 ++++++----------- lib/remote/httputility.hpp | 4 +++- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/lib/remote/httphandler.cpp b/lib/remote/httphandler.cpp index bf8bed37d..d6ea3426b 100644 --- a/lib/remote/httphandler.cpp +++ b/lib/remote/httphandler.cpp @@ -5,6 +5,7 @@ #include "base/singleton.hpp" #include "base/exception.hpp" #include +#include using namespace icinga; @@ -44,11 +45,17 @@ void HttpHandler::Register(const Url::Ptr& url, const HttpHandler::Ptr& handler) handlers->Add(handler); } -void HttpHandler::ProcessRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response) +void HttpHandler::ProcessRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + boost::beast::http::response& response +) { Dictionary::Ptr node = m_UrlTree; std::vector handlers; - const std::vector& path = request.RequestUrl->GetPath(); + + Url::Ptr url = new Url(request.target().to_string()); + auto& path (url->GetPath()); for (std::vector::size_type i = 0; i <= path.size(); i++) { Array::Ptr current_handlers = node->Get("handlers"); @@ -81,7 +88,7 @@ void HttpHandler::ProcessRequest(const ApiUser::Ptr& user, HttpRequest& request, Dictionary::Ptr params; try { - params = HttpUtility::FetchRequestParameters(request); + params = HttpUtility::FetchRequestParameters(url, request.body()); } catch (const std::exception& ex) { HttpUtility::SendJsonError(response, params, 400, "Invalid request body: " + DiagnosticInformation(ex, false)); return; @@ -89,16 +96,15 @@ void HttpHandler::ProcessRequest(const ApiUser::Ptr& user, HttpRequest& request, bool processed = false; for (const HttpHandler::Ptr& handler : handlers) { - if (handler->HandleRequest(user, request, response, params)) { + if (handler->HandleRequest(user, request, url, response, params)) { processed = true; break; } } if (!processed) { - String path = boost::algorithm::join(request.RequestUrl->GetPath(), "/"); - HttpUtility::SendJsonError(response, params, 404, "The requested path '" + path + - "' could not be found or the request method is not valid for this path."); + HttpUtility::SendJsonError(response, params, 404, "The requested path '" + boost::algorithm::join(path, "/") + + "' could not be found or the request method is not valid for this path."); return; } } diff --git a/lib/remote/httphandler.hpp b/lib/remote/httphandler.hpp index 95ddd0b49..7e74329e0 100644 --- a/lib/remote/httphandler.hpp +++ b/lib/remote/httphandler.hpp @@ -4,10 +4,12 @@ #define HTTPHANDLER_H #include "remote/i2-remote.hpp" +#include "remote/url.hpp" #include "remote/httpresponse.hpp" #include "remote/apiuser.hpp" #include "base/registry.hpp" #include +#include namespace icinga { @@ -22,10 +24,20 @@ class HttpHandler : public Object public: DECLARE_PTR_TYPEDEFS(HttpHandler); - virtual bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) = 0; + virtual bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) = 0; static void Register(const Url::Ptr& url, const HttpHandler::Ptr& handler); - static void ProcessRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response); + static void ProcessRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + boost::beast::http::response& response + ); private: static Dictionary::Ptr m_UrlTree; diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 707c1a635..87db3da5c 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -321,7 +321,20 @@ void ProcessRequest( { namespace http = boost::beast::http; - HttpUtility::SendJsonError(response, nullptr, 503, "Unhandled exception" , ""); + try { + CpuBoundWork handlingRequest (yc); + + HttpHandler::ProcessRequest(authenticatedUser, request, response); + } catch (const std::exception& ex) { + http::response response; + + HttpUtility::SendJsonError(response, nullptr, 500, "Unhandled exception" , DiagnosticInformation(ex)); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + + return; + } http::async_write(stream, response, yc); stream.async_flush(yc); diff --git a/lib/remote/httputility.cpp b/lib/remote/httputility.cpp index fa54d9c89..c97297cd0 100644 --- a/lib/remote/httputility.cpp +++ b/lib/remote/httputility.cpp @@ -1,28 +1,23 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "remote/httputility.hpp" +#include "remote/url.hpp" #include "base/json.hpp" #include "base/logger.hpp" #include +#include #include #include using namespace icinga; -Dictionary::Ptr HttpUtility::FetchRequestParameters(HttpRequest& request) +Dictionary::Ptr HttpUtility::FetchRequestParameters(const Url::Ptr& url, const std::string& body) { Dictionary::Ptr result; - String body; - char buffer[1024]; - size_t count; - - while ((count = request.ReadBody(buffer, sizeof(buffer))) > 0) - body += String(buffer, buffer + count); - - if (!body.IsEmpty()) { + if (!body.empty()) { Log(LogDebug, "HttpUtility") - << "Request body: '" << body << "'"; + << "Request body: '" << body << '\''; result = JsonDecode(body); } @@ -31,7 +26,7 @@ Dictionary::Ptr HttpUtility::FetchRequestParameters(HttpRequest& request) result = new Dictionary(); std::map> query; - for (const auto& kv : request.RequestUrl->GetQuery()) { + for (const auto& kv : url->GetQuery()) { query[kv.first].emplace_back(kv.second); } diff --git a/lib/remote/httputility.hpp b/lib/remote/httputility.hpp index 7122e3e71..e9f44b3aa 100644 --- a/lib/remote/httputility.hpp +++ b/lib/remote/httputility.hpp @@ -5,7 +5,9 @@ #include "remote/httprequest.hpp" #include "remote/httpresponse.hpp" +#include "remote/url.hpp" #include "base/dictionary.hpp" +#include #include namespace icinga @@ -20,7 +22,7 @@ class HttpUtility { public: - static Dictionary::Ptr FetchRequestParameters(HttpRequest& request); + static Dictionary::Ptr FetchRequestParameters(const Url::Ptr& url, const std::string& body); static void SendJsonBody(HttpResponse& response, const Dictionary::Ptr& params, const Value& val); static void SendJsonBody(boost::beast::http::response& response, const Dictionary::Ptr& params, const Value& val); static Value GetLastParameter(const Dictionary::Ptr& params, const String& key); From 1941c1da280c94b2e81c6d1aa9c793a09798e678 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 10:33:01 +0100 Subject: [PATCH 15/67] Adjust all HTTP handlers (ex. /v1/events) --- lib/remote/actionshandler.cpp | 20 ++++--- lib/remote/actionshandler.hpp | 9 +++- lib/remote/configfileshandler.cpp | 23 +++++--- lib/remote/configfileshandler.hpp | 9 +++- lib/remote/configpackageshandler.cpp | 68 ++++++++++++++++------- lib/remote/configpackageshandler.hpp | 36 ++++++++++--- lib/remote/configstageshandler.cpp | 80 +++++++++++++++++++--------- lib/remote/configstageshandler.hpp | 36 ++++++++++--- lib/remote/consolehandler.cpp | 30 ++++++++--- lib/remote/consolehandler.hpp | 15 ++++-- lib/remote/createobjecthandler.cpp | 24 ++++++--- lib/remote/createobjecthandler.hpp | 9 +++- lib/remote/deleteobjecthandler.cpp | 24 ++++++--- lib/remote/deleteobjecthandler.hpp | 9 +++- lib/remote/eventshandler.cpp | 20 ++++--- lib/remote/eventshandler.hpp | 9 +++- lib/remote/infohandler.cpp | 31 +++++++---- lib/remote/infohandler.hpp | 9 +++- lib/remote/modifyobjecthandler.cpp | 22 +++++--- lib/remote/modifyobjecthandler.hpp | 9 +++- lib/remote/objectqueryhandler.cpp | 22 +++++--- lib/remote/objectqueryhandler.hpp | 9 +++- lib/remote/statushandler.cpp | 20 ++++--- lib/remote/statushandler.hpp | 9 +++- lib/remote/templatequeryhandler.cpp | 22 +++++--- lib/remote/templatequeryhandler.hpp | 9 +++- lib/remote/typequeryhandler.cpp | 20 ++++--- lib/remote/typequeryhandler.hpp | 9 +++- lib/remote/variablequeryhandler.cpp | 20 ++++--- lib/remote/variablequeryhandler.hpp | 9 +++- 30 files changed, 461 insertions(+), 180 deletions(-) diff --git a/lib/remote/actionshandler.cpp b/lib/remote/actionshandler.cpp index cc1cba8de..5d042fe73 100644 --- a/lib/remote/actionshandler.cpp +++ b/lib/remote/actionshandler.cpp @@ -12,15 +12,23 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/actions", ActionsHandler); -bool ActionsHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ActionsHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() != 3) + namespace http = boost::beast::http; + + if (url->GetPath().size() != 3) return false; - if (request.RequestMethod != "POST") + if (request.method() != http::verb::post) return false; - String actionName = request.RequestUrl->GetPath()[2]; + String actionName = url->GetPath()[2]; ApiAction::Ptr action = ApiAction::GetByName(actionName); @@ -81,17 +89,15 @@ bool ActionsHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& reques } int statusCode = 500; - String statusMessage = "No action executed successfully"; for (const Dictionary::Ptr& res : results) { if (res->Contains("code") && res->Get("code") == 200) { statusCode = 200; - statusMessage = "OK"; break; } } - response.SetStatus(statusCode, statusMessage); + response.result(statusCode); Dictionary::Ptr result = new Dictionary({ { "results", new Array(std::move(results)) } diff --git a/lib/remote/actionshandler.hpp b/lib/remote/actionshandler.hpp index 7affb0814..790796ac3 100644 --- a/lib/remote/actionshandler.hpp +++ b/lib/remote/actionshandler.hpp @@ -13,8 +13,13 @@ class ActionsHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ActionsHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/configfileshandler.cpp b/lib/remote/configfileshandler.cpp index abd972825..93b63ecd5 100644 --- a/lib/remote/configfileshandler.cpp +++ b/lib/remote/configfileshandler.cpp @@ -12,12 +12,20 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/files", ConfigFilesHandler); -bool ConfigFilesHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ConfigFilesHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestMethod != "GET") + namespace http = boost::beast::http; + + if (request.method() != http::verb::get) return false; - const std::vector& urlPath = request.RequestUrl->GetPath(); + const std::vector& urlPath = url->GetPath(); if (urlPath.size() >= 4) params->Set("package", urlPath[3]); @@ -30,7 +38,7 @@ bool ConfigFilesHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& re params->Set("path", boost::algorithm::join(tmpPath, "/")); } - if (request.Headers->Get("accept") == "application/json") { + if (request[http::field::accept] == "application/json") { HttpUtility::SendJsonError(response, params, 400, "Invalid Accept header. Either remove the Accept header or set it to 'application/octet-stream'."); return true; } @@ -69,9 +77,10 @@ bool ConfigFilesHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& re fp.exceptions(std::ifstream::badbit); String content((std::istreambuf_iterator(fp)), std::istreambuf_iterator()); - response.SetStatus(200, "OK"); - response.AddHeader("Content-Type", "application/octet-stream"); - response.WriteBody(content.CStr(), content.GetLength()); + response.result(http::status::ok); + response.set(http::field::content_type, "application/octet-stream"); + response.body() = content; + response.set(http::field::content_length, response.body().size()); } catch (const std::exception& ex) { HttpUtility::SendJsonError(response, params, 500, "Could not read file.", DiagnosticInformation(ex)); diff --git a/lib/remote/configfileshandler.hpp b/lib/remote/configfileshandler.hpp index 17f03e0dd..50b294493 100644 --- a/lib/remote/configfileshandler.hpp +++ b/lib/remote/configfileshandler.hpp @@ -13,8 +13,13 @@ class ConfigFilesHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ConfigFilesHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/configpackageshandler.cpp b/lib/remote/configpackageshandler.cpp index b590e24fb..606d232dd 100644 --- a/lib/remote/configpackageshandler.cpp +++ b/lib/remote/configpackageshandler.cpp @@ -10,25 +10,41 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/packages", ConfigPackagesHandler); -bool ConfigPackagesHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ConfigPackagesHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 4) return false; - if (request.RequestMethod == "GET") - HandleGet(user, request, response, params); - else if (request.RequestMethod == "POST") - HandlePost(user, request, response, params); - else if (request.RequestMethod == "DELETE") - HandleDelete(user, request, response, params); + if (request.method() == http::verb::get) + HandleGet(user, request, url, response, params); + else if (request.method() == http::verb::post) + HandlePost(user, request, url, response, params); + else if (request.method() == http::verb::delete_) + HandleDelete(user, request, url, response, params); else return false; return true; } -void ConfigPackagesHandler::HandleGet(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigPackagesHandler::HandleGet( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/query"); std::vector packages; @@ -58,16 +74,24 @@ void ConfigPackagesHandler::HandleGet(const ApiUser::Ptr& user, HttpRequest& req { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } -void ConfigPackagesHandler::HandlePost(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigPackagesHandler::HandlePost( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/modify"); - if (request.RequestUrl->GetPath().size() >= 4) - params->Set("package", request.RequestUrl->GetPath()[3]); + if (url->GetPath().size() >= 4) + params->Set("package", url->GetPath()[3]); String packageName = HttpUtility::GetLastParameter(params, "package"); @@ -95,16 +119,24 @@ void ConfigPackagesHandler::HandlePost(const ApiUser::Ptr& user, HttpRequest& re { "results", new Array({ result1 }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } -void ConfigPackagesHandler::HandleDelete(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigPackagesHandler::HandleDelete( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/modify"); - if (request.RequestUrl->GetPath().size() >= 4) - params->Set("package", request.RequestUrl->GetPath()[3]); + if (url->GetPath().size() >= 4) + params->Set("package", url->GetPath()[3]); String packageName = HttpUtility::GetLastParameter(params, "package"); @@ -131,6 +163,6 @@ void ConfigPackagesHandler::HandleDelete(const ApiUser::Ptr& user, HttpRequest& { "results", new Array({ result1 }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } diff --git a/lib/remote/configpackageshandler.hpp b/lib/remote/configpackageshandler.hpp index 12c081afe..395b778eb 100644 --- a/lib/remote/configpackageshandler.hpp +++ b/lib/remote/configpackageshandler.hpp @@ -13,16 +13,36 @@ class ConfigPackagesHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ConfigPackagesHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; private: - void HandleGet(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); - void HandlePost(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); - void HandleDelete(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); + void HandleGet( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); + void HandlePost( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); + void HandleDelete( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); }; diff --git a/lib/remote/configstageshandler.cpp b/lib/remote/configstageshandler.cpp index fa0078c39..2254511e1 100644 --- a/lib/remote/configstageshandler.cpp +++ b/lib/remote/configstageshandler.cpp @@ -11,32 +11,48 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/stages", ConfigStagesHandler); -bool ConfigStagesHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ConfigStagesHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 5) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 5) return false; - if (request.RequestMethod == "GET") - HandleGet(user, request, response, params); - else if (request.RequestMethod == "POST") - HandlePost(user, request, response, params); - else if (request.RequestMethod == "DELETE") - HandleDelete(user, request, response, params); + if (request.method() == http::verb::get) + HandleGet(user, request, url, response, params); + else if (request.method() == http::verb::post) + HandlePost(user, request, url, response, params); + else if (request.method() == http::verb::delete_) + HandleDelete(user, request, url, response, params); else return false; return true; } -void ConfigStagesHandler::HandleGet(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigStagesHandler::HandleGet( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/query"); - if (request.RequestUrl->GetPath().size() >= 4) - params->Set("package", request.RequestUrl->GetPath()[3]); + if (url->GetPath().size() >= 4) + params->Set("package", url->GetPath()[3]); - if (request.RequestUrl->GetPath().size() >= 5) - params->Set("stage", request.RequestUrl->GetPath()[4]); + if (url->GetPath().size() >= 5) + params->Set("stage", url->GetPath()[4]); String packageName = HttpUtility::GetLastParameter(params, "package"); String stageName = HttpUtility::GetLastParameter(params, "stage"); @@ -64,16 +80,24 @@ void ConfigStagesHandler::HandleGet(const ApiUser::Ptr& user, HttpRequest& reque { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } -void ConfigStagesHandler::HandlePost(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigStagesHandler::HandlePost( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/modify"); - if (request.RequestUrl->GetPath().size() >= 4) - params->Set("package", request.RequestUrl->GetPath()[3]); + if (url->GetPath().size() >= 4) + params->Set("package", url->GetPath()[3]); String packageName = HttpUtility::GetLastParameter(params, "package"); @@ -123,19 +147,27 @@ void ConfigStagesHandler::HandlePost(const ApiUser::Ptr& user, HttpRequest& requ { "results", new Array({ result1 }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } -void ConfigStagesHandler::HandleDelete(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +void ConfigStagesHandler::HandleDelete( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { + namespace http = boost::beast::http; + FilterUtility::CheckPermission(user, "config/modify"); - if (request.RequestUrl->GetPath().size() >= 4) - params->Set("package", request.RequestUrl->GetPath()[3]); + if (url->GetPath().size() >= 4) + params->Set("package", url->GetPath()[3]); - if (request.RequestUrl->GetPath().size() >= 5) - params->Set("stage", request.RequestUrl->GetPath()[4]); + if (url->GetPath().size() >= 5) + params->Set("stage", url->GetPath()[4]); String packageName = HttpUtility::GetLastParameter(params, "package"); String stageName = HttpUtility::GetLastParameter(params, "stage"); @@ -165,7 +197,7 @@ void ConfigStagesHandler::HandleDelete(const ApiUser::Ptr& user, HttpRequest& re { "results", new Array({ result1 }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); } diff --git a/lib/remote/configstageshandler.hpp b/lib/remote/configstageshandler.hpp index a5ac21605..aec7d4eee 100644 --- a/lib/remote/configstageshandler.hpp +++ b/lib/remote/configstageshandler.hpp @@ -13,16 +13,36 @@ class ConfigStagesHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ConfigStagesHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; private: - void HandleGet(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); - void HandlePost(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); - void HandleDelete(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params); + void HandleGet( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); + void HandlePost( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); + void HandleDelete( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ); }; diff --git a/lib/remote/consolehandler.cpp b/lib/remote/consolehandler.cpp index 67a4988d9..f513b21c0 100644 --- a/lib/remote/consolehandler.cpp +++ b/lib/remote/consolehandler.cpp @@ -53,17 +53,25 @@ static void EnsureFrameCleanupTimer() }); } -bool ConsoleHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ConsoleHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() != 3) + namespace http = boost::beast::http; + + if (url->GetPath().size() != 3) return false; - if (request.RequestMethod != "POST") + if (request.method() != http::verb::post) return false; QueryDescription qd; - String methodName = request.RequestUrl->GetPath()[2]; + String methodName = url->GetPath()[2]; FilterUtility::CheckPermission(user, "console"); @@ -85,9 +93,12 @@ bool ConsoleHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& reques return true; } -bool ConsoleHandler::ExecuteScriptHelper(HttpRequest& request, HttpResponse& response, +bool ConsoleHandler::ExecuteScriptHelper(boost::beast::http::request& request, + boost::beast::http::response& response, const Dictionary::Ptr& params, const String& command, const String& session, bool sandboxed) { + namespace http = boost::beast::http; + Log(LogNotice, "Console") << "Executing expression: " << command; @@ -151,15 +162,18 @@ bool ConsoleHandler::ExecuteScriptHelper(HttpRequest& request, HttpResponse& res { "results", new Array({ resultInfo }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; } -bool ConsoleHandler::AutocompleteScriptHelper(HttpRequest& request, HttpResponse& response, +bool ConsoleHandler::AutocompleteScriptHelper(boost::beast::http::request& request, + boost::beast::http::response& response, const Dictionary::Ptr& params, const String& command, const String& session, bool sandboxed) { + namespace http = boost::beast::http; + Log(LogInformation, "Console") << "Auto-completing expression: " << command; @@ -187,7 +201,7 @@ bool ConsoleHandler::AutocompleteScriptHelper(HttpRequest& request, HttpResponse { "results", new Array({ result1 }) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/consolehandler.hpp b/lib/remote/consolehandler.hpp index 5ef22ca1a..0c0a12399 100644 --- a/lib/remote/consolehandler.hpp +++ b/lib/remote/consolehandler.hpp @@ -22,15 +22,22 @@ class ConsoleHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ConsoleHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; static std::vector GetAutocompletionSuggestions(const String& word, ScriptFrame& frame); private: - static bool ExecuteScriptHelper(HttpRequest& request, HttpResponse& response, + static bool ExecuteScriptHelper(boost::beast::http::request& request, + boost::beast::http::response& response, const Dictionary::Ptr& params, const String& command, const String& session, bool sandboxed); - static bool AutocompleteScriptHelper(HttpRequest& request, HttpResponse& response, + static bool AutocompleteScriptHelper(boost::beast::http::request& request, + boost::beast::http::response& response, const Dictionary::Ptr& params, const String& command, const String& session, bool sandboxed); }; diff --git a/lib/remote/createobjecthandler.cpp b/lib/remote/createobjecthandler.cpp index 30eb0bac7..bf9cb990e 100644 --- a/lib/remote/createobjecthandler.cpp +++ b/lib/remote/createobjecthandler.cpp @@ -14,15 +14,23 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", CreateObjectHandler); -bool CreateObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool CreateObjectHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() != 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() != 4) return false; - if (request.RequestMethod != "PUT") + if (request.method() != http::verb::put) return false; - Type::Ptr type = FilterUtility::TypeFromPluralName(request.RequestUrl->GetPath()[2]); + Type::Ptr type = FilterUtility::TypeFromPluralName(url->GetPath()[2]); if (!type) { HttpUtility::SendJsonError(response, params, 400, "Invalid type specified."); @@ -31,7 +39,7 @@ bool CreateObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r FilterUtility::CheckPermission(user, "objects/create/" + type->GetName()); - String name = request.RequestUrl->GetPath()[3]; + String name = url->GetPath()[3]; Array::Ptr templates = params->Get("templates"); Dictionary::Ptr attrs = params->Get("attrs"); @@ -99,7 +107,7 @@ bool CreateObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r result1->Set("code", 500); result1->Set("status", "Object could not be created."); - response.SetStatus(500, "Object could not be created"); + response.result(http::status::internal_server_error); HttpUtility::SendJsonBody(response, params, result); return true; @@ -113,7 +121,7 @@ bool CreateObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r if (verbose) result1->Set("diagnostic_information", diagnosticInformation); - response.SetStatus(500, "Object could not be created"); + response.result(http::status::internal_server_error); HttpUtility::SendJsonBody(response, params, result); return true; @@ -129,7 +137,7 @@ bool CreateObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r else if (!obj && ignoreOnError) result1->Set("status", "Object was not created but 'ignore_on_error' was set to true"); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/createobjecthandler.hpp b/lib/remote/createobjecthandler.hpp index e9da35abf..646b07fc6 100644 --- a/lib/remote/createobjecthandler.hpp +++ b/lib/remote/createobjecthandler.hpp @@ -13,8 +13,13 @@ class CreateObjectHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(CreateObjectHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/deleteobjecthandler.cpp b/lib/remote/deleteobjecthandler.cpp index 3c2f7c1a8..996effea8 100644 --- a/lib/remote/deleteobjecthandler.cpp +++ b/lib/remote/deleteobjecthandler.cpp @@ -14,15 +14,23 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", DeleteObjectHandler); -bool DeleteObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool DeleteObjectHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() < 3 || request.RequestUrl->GetPath().size() > 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() < 3 || url->GetPath().size() > 4) return false; - if (request.RequestMethod != "DELETE") + if (request.method() != http::verb::delete_) return false; - Type::Ptr type = FilterUtility::TypeFromPluralName(request.RequestUrl->GetPath()[2]); + Type::Ptr type = FilterUtility::TypeFromPluralName(url->GetPath()[2]); if (!type) { HttpUtility::SendJsonError(response, params, 400, "Invalid type specified."); @@ -35,10 +43,10 @@ bool DeleteObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r params->Set("type", type->GetName()); - if (request.RequestUrl->GetPath().size() >= 4) { + if (url->GetPath().size() >= 4) { String attr = type->GetName(); boost::algorithm::to_lower(attr); - params->Set(attr, request.RequestUrl->GetPath()[3]); + params->Set(attr, url->GetPath()[3]); } std::vector objs; @@ -93,9 +101,9 @@ bool DeleteObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r }); if (!success) - response.SetStatus(500, "One or more objects could not be deleted"); + response.result(http::status::internal_server_error); else - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); diff --git a/lib/remote/deleteobjecthandler.hpp b/lib/remote/deleteobjecthandler.hpp index 61cfa8bf7..97fbbc5be 100644 --- a/lib/remote/deleteobjecthandler.hpp +++ b/lib/remote/deleteobjecthandler.hpp @@ -13,8 +13,13 @@ class DeleteObjectHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(DeleteObjectHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index 7208e3d6b..ed829706d 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -13,15 +13,23 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/events", EventsHandler); -bool EventsHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool EventsHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() != 2) + namespace http = boost::beast::http; + + if (url->GetPath().size() != 2) return false; - if (request.RequestMethod != "POST") + if (request.method() != http::verb::post) return false; - if (request.ProtocolVersion == HttpVersion10) { + if (request.version() == 10) { HttpUtility::SendJsonError(response, params, 400, "HTTP/1.0 not supported for event streams."); return true; } @@ -67,8 +75,8 @@ bool EventsHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request queue->AddClient(&request); - response.SetStatus(200, "OK"); - response.AddHeader("Content-Type", "application/json"); + response.result(http::status::ok); + response.set(http::field::content_type, "application/json"); for (;;) { Dictionary::Ptr result = queue->WaitForEvent(&request); diff --git a/lib/remote/eventshandler.hpp b/lib/remote/eventshandler.hpp index 78f8d03d0..d95d31bf1 100644 --- a/lib/remote/eventshandler.hpp +++ b/lib/remote/eventshandler.hpp @@ -14,8 +14,13 @@ class EventsHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(EventsHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/infohandler.cpp b/lib/remote/infohandler.cpp index 6925a668f..c605cca8b 100644 --- a/lib/remote/infohandler.cpp +++ b/lib/remote/infohandler.cpp @@ -8,24 +8,32 @@ using namespace icinga; REGISTER_URLHANDLER("/", InfoHandler); -bool InfoHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool InfoHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 2) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 2) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; - if (request.RequestUrl->GetPath().empty()) { - response.SetStatus(302, "Found"); - response.AddHeader("Location", "/v1"); + if (url->GetPath().empty()) { + response.result(http::status::found); + response.set(http::field::location, "/v1"); return true; } - if (request.RequestUrl->GetPath()[0] != "v1" || request.RequestUrl->GetPath().size() != 1) + if (url->GetPath()[0] != "v1" || url->GetPath().size() != 1) return false; - response.SetStatus(200, "OK"); + response.result(http::status::ok); std::vector permInfo; Array::Ptr permissions = user->GetPermissions(); @@ -49,7 +57,7 @@ bool InfoHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, } } - if (request.Headers->Get("accept") == "application/json") { + if (request[http::field::accept] == "application/json") { Dictionary::Ptr result1 = new Dictionary({ { "user", user->GetName() }, { "permissions", Array::FromVector(permInfo) }, @@ -63,7 +71,7 @@ bool InfoHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpUtility::SendJsonBody(response, params, result); } else { - response.AddHeader("Content-Type", "text/html"); + response.set(http::field::content_type, "text/html"); String body = "Icinga 2

Hello from Icinga 2 (Version: " + Application::GetAppVersion() + ")!

"; body += "

You are authenticated as " + user->GetName() + ". "; @@ -80,7 +88,8 @@ bool InfoHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, body += "Your user does not have any permissions.

"; body += R"(

More information about API requests is available in the documentation.

)"; - response.WriteBody(body.CStr(), body.GetLength()); + response.body() = body; + response.set(http::field::content_length, response.body().size()); } return true; diff --git a/lib/remote/infohandler.hpp b/lib/remote/infohandler.hpp index 38c88af11..418bb9b4b 100644 --- a/lib/remote/infohandler.hpp +++ b/lib/remote/infohandler.hpp @@ -13,8 +13,13 @@ class InfoHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(InfoHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/modifyobjecthandler.cpp b/lib/remote/modifyobjecthandler.cpp index 8e93f7423..af800cce1 100644 --- a/lib/remote/modifyobjecthandler.cpp +++ b/lib/remote/modifyobjecthandler.cpp @@ -12,15 +12,23 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", ModifyObjectHandler); -bool ModifyObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ModifyObjectHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() < 3 || request.RequestUrl->GetPath().size() > 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() < 3 || url->GetPath().size() > 4) return false; - if (request.RequestMethod != "POST") + if (request.method() != http::verb::post) return false; - Type::Ptr type = FilterUtility::TypeFromPluralName(request.RequestUrl->GetPath()[2]); + Type::Ptr type = FilterUtility::TypeFromPluralName(url->GetPath()[2]); if (!type) { HttpUtility::SendJsonError(response, params, 400, "Invalid type specified."); @@ -33,10 +41,10 @@ bool ModifyObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r params->Set("type", type->GetName()); - if (request.RequestUrl->GetPath().size() >= 4) { + if (url->GetPath().size() >= 4) { String attr = type->GetName(); boost::algorithm::to_lower(attr); - params->Set(attr, request.RequestUrl->GetPath()[3]); + params->Set(attr, url->GetPath()[3]); } std::vector objs; @@ -101,7 +109,7 @@ bool ModifyObjectHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& r { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/modifyobjecthandler.hpp b/lib/remote/modifyobjecthandler.hpp index 2df23656c..e63304beb 100644 --- a/lib/remote/modifyobjecthandler.hpp +++ b/lib/remote/modifyobjecthandler.hpp @@ -13,8 +13,13 @@ class ModifyObjectHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ModifyObjectHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/objectqueryhandler.cpp b/lib/remote/objectqueryhandler.cpp index 132e2bddd..63a67d853 100644 --- a/lib/remote/objectqueryhandler.cpp +++ b/lib/remote/objectqueryhandler.cpp @@ -87,15 +87,23 @@ Dictionary::Ptr ObjectQueryHandler::SerializeObjectAttrs(const Object::Ptr& obje return new Dictionary(std::move(resultAttrs)); } -bool ObjectQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool ObjectQueryHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() < 3 || request.RequestUrl->GetPath().size() > 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() < 3 || url->GetPath().size() > 4) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; - Type::Ptr type = FilterUtility::TypeFromPluralName(request.RequestUrl->GetPath()[2]); + Type::Ptr type = FilterUtility::TypeFromPluralName(url->GetPath()[2]); if (!type) { HttpUtility::SendJsonError(response, params, 400, "Invalid type specified."); @@ -136,10 +144,10 @@ bool ObjectQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& re params->Set("type", type->GetName()); - if (request.RequestUrl->GetPath().size() >= 4) { + if (url->GetPath().size() >= 4) { String attr = type->GetName(); boost::algorithm::to_lower(attr); - params->Set(attr, request.RequestUrl->GetPath()[3]); + params->Set(attr, url->GetPath()[3]); } std::vector objs; @@ -265,7 +273,7 @@ bool ObjectQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& re { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/objectqueryhandler.hpp b/lib/remote/objectqueryhandler.hpp index 634775c64..86d3ef878 100644 --- a/lib/remote/objectqueryhandler.hpp +++ b/lib/remote/objectqueryhandler.hpp @@ -13,8 +13,13 @@ class ObjectQueryHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(ObjectQueryHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; private: static Dictionary::Ptr SerializeObjectAttrs(const Object::Ptr& object, const String& attrPrefix, diff --git a/lib/remote/statushandler.cpp b/lib/remote/statushandler.cpp index e40a7ae03..9b1175ba4 100644 --- a/lib/remote/statushandler.cpp +++ b/lib/remote/statushandler.cpp @@ -68,12 +68,20 @@ public: } }; -bool StatusHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool StatusHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 3) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 3) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; QueryDescription qd; @@ -83,8 +91,8 @@ bool StatusHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request params->Set("type", "Status"); - if (request.RequestUrl->GetPath().size() >= 3) - params->Set("status", request.RequestUrl->GetPath()[2]); + if (url->GetPath().size() >= 3) + params->Set("status", url->GetPath()[2]); std::vector objs; @@ -101,7 +109,7 @@ bool StatusHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request { "results", new Array(std::move(objs)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/statushandler.hpp b/lib/remote/statushandler.hpp index a4ec52875..a26161612 100644 --- a/lib/remote/statushandler.hpp +++ b/lib/remote/statushandler.hpp @@ -13,8 +13,13 @@ class StatusHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(StatusHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/templatequeryhandler.cpp b/lib/remote/templatequeryhandler.cpp index ea24fd20f..9508163b1 100644 --- a/lib/remote/templatequeryhandler.cpp +++ b/lib/remote/templatequeryhandler.cpp @@ -75,15 +75,23 @@ public: } }; -bool TemplateQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool TemplateQueryHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() < 3 || request.RequestUrl->GetPath().size() > 4) + namespace http = boost::beast::http; + + if (url->GetPath().size() < 3 || url->GetPath().size() > 4) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; - Type::Ptr type = FilterUtility::TypeFromPluralName(request.RequestUrl->GetPath()[2]); + Type::Ptr type = FilterUtility::TypeFromPluralName(url->GetPath()[2]); if (!type) { HttpUtility::SendJsonError(response, params, 400, "Invalid type specified."); @@ -97,10 +105,10 @@ bool TemplateQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& params->Set("type", type->GetName()); - if (request.RequestUrl->GetPath().size() >= 4) { + if (url->GetPath().size() >= 4) { String attr = type->GetName(); boost::algorithm::to_lower(attr); - params->Set(attr, request.RequestUrl->GetPath()[3]); + params->Set(attr, url->GetPath()[3]); } std::vector objs; @@ -118,7 +126,7 @@ bool TemplateQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& { "results", new Array(std::move(objs)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/templatequeryhandler.hpp b/lib/remote/templatequeryhandler.hpp index 4e137e943..e98bcd249 100644 --- a/lib/remote/templatequeryhandler.hpp +++ b/lib/remote/templatequeryhandler.hpp @@ -13,8 +13,13 @@ class TemplateQueryHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(TemplateQueryHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/typequeryhandler.cpp b/lib/remote/typequeryhandler.cpp index e3972616a..70c83f7e7 100644 --- a/lib/remote/typequeryhandler.cpp +++ b/lib/remote/typequeryhandler.cpp @@ -46,12 +46,20 @@ public: } }; -bool TypeQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool TypeQueryHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 3) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 3) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; QueryDescription qd; @@ -64,8 +72,8 @@ bool TypeQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& requ params->Set("type", "Type"); - if (request.RequestUrl->GetPath().size() >= 3) - params->Set("name", request.RequestUrl->GetPath()[2]); + if (url->GetPath().size() >= 3) + params->Set("name", url->GetPath()[2]); std::vector objs; @@ -138,7 +146,7 @@ bool TypeQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& requ { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/typequeryhandler.hpp b/lib/remote/typequeryhandler.hpp index 4ca3fcf4e..69d65baec 100644 --- a/lib/remote/typequeryhandler.hpp +++ b/lib/remote/typequeryhandler.hpp @@ -13,8 +13,13 @@ class TypeQueryHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(TypeQueryHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } diff --git a/lib/remote/variablequeryhandler.cpp b/lib/remote/variablequeryhandler.cpp index 04d014784..dcf01d3e9 100644 --- a/lib/remote/variablequeryhandler.cpp +++ b/lib/remote/variablequeryhandler.cpp @@ -56,12 +56,20 @@ public: } }; -bool VariableQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, HttpResponse& response, const Dictionary::Ptr& params) +bool VariableQueryHandler::HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params +) { - if (request.RequestUrl->GetPath().size() > 3) + namespace http = boost::beast::http; + + if (url->GetPath().size() > 3) return false; - if (request.RequestMethod != "GET") + if (request.method() != http::verb::get) return false; QueryDescription qd; @@ -71,8 +79,8 @@ bool VariableQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& params->Set("type", "Variable"); - if (request.RequestUrl->GetPath().size() >= 3) - params->Set("variable", request.RequestUrl->GetPath()[2]); + if (url->GetPath().size() >= 3) + params->Set("variable", url->GetPath()[2]); std::vector objs; @@ -99,7 +107,7 @@ bool VariableQueryHandler::HandleRequest(const ApiUser::Ptr& user, HttpRequest& { "results", new Array(std::move(results)) } }); - response.SetStatus(200, "OK"); + response.result(http::status::ok); HttpUtility::SendJsonBody(response, params, result); return true; diff --git a/lib/remote/variablequeryhandler.hpp b/lib/remote/variablequeryhandler.hpp index 85a006a4b..5da2e13ab 100644 --- a/lib/remote/variablequeryhandler.hpp +++ b/lib/remote/variablequeryhandler.hpp @@ -13,8 +13,13 @@ class VariableQueryHandler final : public HttpHandler public: DECLARE_PTR_TYPEDEFS(VariableQueryHandler); - bool HandleRequest(const ApiUser::Ptr& user, HttpRequest& request, - HttpResponse& response, const Dictionary::Ptr& params) override; + bool HandleRequest( + const ApiUser::Ptr& user, + boost::beast::http::request& request, + const Url::Ptr& url, + boost::beast::http::response& response, + const Dictionary::Ptr& params + ) override; }; } From fd239ba3feed5ae7148cde251fcabd2cfcce214e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 11:51:12 +0100 Subject: [PATCH 16/67] Adjust /v1/events, too --- lib/remote/actionshandler.cpp | 5 +++- lib/remote/actionshandler.hpp | 5 +++- lib/remote/configfileshandler.cpp | 5 +++- lib/remote/configfileshandler.hpp | 5 +++- lib/remote/configpackageshandler.cpp | 5 +++- lib/remote/configpackageshandler.hpp | 5 +++- lib/remote/configstageshandler.cpp | 5 +++- lib/remote/configstageshandler.hpp | 5 +++- lib/remote/consolehandler.cpp | 5 +++- lib/remote/consolehandler.hpp | 5 +++- lib/remote/createobjecthandler.cpp | 5 +++- lib/remote/createobjecthandler.hpp | 5 +++- lib/remote/deleteobjecthandler.cpp | 5 +++- lib/remote/deleteobjecthandler.hpp | 5 +++- lib/remote/eventshandler.cpp | 39 +++++++++++++++++----------- lib/remote/eventshandler.hpp | 5 +++- lib/remote/httphandler.cpp | 7 +++-- lib/remote/httphandler.hpp | 12 +++++++-- lib/remote/httpserverconnection.cpp | 22 +++++++++++++--- lib/remote/infohandler.cpp | 5 +++- lib/remote/infohandler.hpp | 5 +++- lib/remote/modifyobjecthandler.cpp | 5 +++- lib/remote/modifyobjecthandler.hpp | 5 +++- lib/remote/objectqueryhandler.cpp | 5 +++- lib/remote/objectqueryhandler.hpp | 5 +++- lib/remote/statushandler.cpp | 5 +++- lib/remote/statushandler.hpp | 5 +++- lib/remote/templatequeryhandler.cpp | 5 +++- lib/remote/templatequeryhandler.hpp | 5 +++- lib/remote/typequeryhandler.cpp | 5 +++- lib/remote/typequeryhandler.hpp | 5 +++- lib/remote/variablequeryhandler.cpp | 5 +++- lib/remote/variablequeryhandler.hpp | 5 +++- 33 files changed, 173 insertions(+), 52 deletions(-) diff --git a/lib/remote/actionshandler.cpp b/lib/remote/actionshandler.cpp index 5d042fe73..251f8a82e 100644 --- a/lib/remote/actionshandler.cpp +++ b/lib/remote/actionshandler.cpp @@ -13,11 +13,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/actions", ActionsHandler); bool ActionsHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/actionshandler.hpp b/lib/remote/actionshandler.hpp index 790796ac3..9c0463035 100644 --- a/lib/remote/actionshandler.hpp +++ b/lib/remote/actionshandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ActionsHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/configfileshandler.cpp b/lib/remote/configfileshandler.cpp index 93b63ecd5..05def8655 100644 --- a/lib/remote/configfileshandler.cpp +++ b/lib/remote/configfileshandler.cpp @@ -13,11 +13,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/files", ConfigFilesHandler); bool ConfigFilesHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/configfileshandler.hpp b/lib/remote/configfileshandler.hpp index 50b294493..1384c2b58 100644 --- a/lib/remote/configfileshandler.hpp +++ b/lib/remote/configfileshandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ConfigFilesHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/configpackageshandler.cpp b/lib/remote/configpackageshandler.cpp index 606d232dd..2505de36a 100644 --- a/lib/remote/configpackageshandler.cpp +++ b/lib/remote/configpackageshandler.cpp @@ -11,11 +11,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/packages", ConfigPackagesHandler); bool ConfigPackagesHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/configpackageshandler.hpp b/lib/remote/configpackageshandler.hpp index 395b778eb..2ae184c6a 100644 --- a/lib/remote/configpackageshandler.hpp +++ b/lib/remote/configpackageshandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ConfigPackagesHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; private: diff --git a/lib/remote/configstageshandler.cpp b/lib/remote/configstageshandler.cpp index 2254511e1..067181f27 100644 --- a/lib/remote/configstageshandler.cpp +++ b/lib/remote/configstageshandler.cpp @@ -12,11 +12,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/stages", ConfigStagesHandler); bool ConfigStagesHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/configstageshandler.hpp b/lib/remote/configstageshandler.hpp index aec7d4eee..4ee35ad91 100644 --- a/lib/remote/configstageshandler.hpp +++ b/lib/remote/configstageshandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ConfigStagesHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; private: diff --git a/lib/remote/consolehandler.cpp b/lib/remote/consolehandler.cpp index f513b21c0..430a37cf3 100644 --- a/lib/remote/consolehandler.cpp +++ b/lib/remote/consolehandler.cpp @@ -54,11 +54,14 @@ static void EnsureFrameCleanupTimer() } bool ConsoleHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/consolehandler.hpp b/lib/remote/consolehandler.hpp index 0c0a12399..339f3ed86 100644 --- a/lib/remote/consolehandler.hpp +++ b/lib/remote/consolehandler.hpp @@ -23,11 +23,14 @@ public: DECLARE_PTR_TYPEDEFS(ConsoleHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; static std::vector GetAutocompletionSuggestions(const String& word, ScriptFrame& frame); diff --git a/lib/remote/createobjecthandler.cpp b/lib/remote/createobjecthandler.cpp index bf9cb990e..1f7ab6710 100644 --- a/lib/remote/createobjecthandler.cpp +++ b/lib/remote/createobjecthandler.cpp @@ -15,11 +15,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", CreateObjectHandler); bool CreateObjectHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/createobjecthandler.hpp b/lib/remote/createobjecthandler.hpp index 646b07fc6..ff42a62e6 100644 --- a/lib/remote/createobjecthandler.hpp +++ b/lib/remote/createobjecthandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(CreateObjectHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/deleteobjecthandler.cpp b/lib/remote/deleteobjecthandler.cpp index 996effea8..15951588e 100644 --- a/lib/remote/deleteobjecthandler.cpp +++ b/lib/remote/deleteobjecthandler.cpp @@ -15,11 +15,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", DeleteObjectHandler); bool DeleteObjectHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/deleteobjecthandler.hpp b/lib/remote/deleteobjecthandler.hpp index 97fbbc5be..3ecb0c71b 100644 --- a/lib/remote/deleteobjecthandler.hpp +++ b/lib/remote/deleteobjecthandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(DeleteObjectHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index ed829706d..99843d481 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -5,8 +5,10 @@ #include "remote/filterutility.hpp" #include "config/configcompiler.hpp" #include "config/expression.hpp" +#include "base/defer.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" +#include #include using namespace icinga; @@ -14,13 +16,17 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/events", EventsHandler); bool EventsHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { + namespace asio = boost::asio; namespace http = boost::beast::http; if (url->GetPath().size() != 2) @@ -75,18 +81,24 @@ bool EventsHandler::HandleRequest( queue->AddClient(&request); + Defer removeClient ([&queue, &request, &queueName]() { + queue->RemoveClient(&request); + EventQueue::UnregisterIfUnused(queueName, queue); + }); + + hasStartedStreaming = true; + response.result(http::status::ok); response.set(http::field::content_type, "application/json"); + http::async_write(stream, response, yc); + stream.async_flush(yc); + + asio::const_buffer newLine ("\n", 1); + for (;;) { Dictionary::Ptr result = queue->WaitForEvent(&request); - if (!response.IsPeerConnected()) { - queue->RemoveClient(&request); - EventQueue::UnregisterIfUnused(queueName, queue); - return true; - } - if (!result) continue; @@ -94,14 +106,11 @@ bool EventsHandler::HandleRequest( boost::algorithm::replace_all(body, "\n", ""); - try { - response.WriteBody(body.CStr(), body.GetLength()); - response.WriteBody("\n", 1); - } catch (const std::exception&) { - queue->RemoveClient(&request); - EventQueue::UnregisterIfUnused(queueName, queue); - throw; - } + asio::const_buffer payload (body.CStr(), body.GetLength()); + + stream.async_write_some(payload, yc); + stream.async_write_some(newLine, yc); + stream.async_flush(yc); } } diff --git a/lib/remote/eventshandler.hpp b/lib/remote/eventshandler.hpp index d95d31bf1..cffeea503 100644 --- a/lib/remote/eventshandler.hpp +++ b/lib/remote/eventshandler.hpp @@ -15,11 +15,14 @@ public: DECLARE_PTR_TYPEDEFS(EventsHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/httphandler.cpp b/lib/remote/httphandler.cpp index d6ea3426b..c089c00ea 100644 --- a/lib/remote/httphandler.cpp +++ b/lib/remote/httphandler.cpp @@ -46,9 +46,12 @@ void HttpHandler::Register(const Url::Ptr& url, const HttpHandler::Ptr& handler) } void HttpHandler::ProcessRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, - boost::beast::http::response& response + boost::beast::http::response& response, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { Dictionary::Ptr node = m_UrlTree; @@ -96,7 +99,7 @@ void HttpHandler::ProcessRequest( bool processed = false; for (const HttpHandler::Ptr& handler : handlers) { - if (handler->HandleRequest(user, request, url, response, params)) { + if (handler->HandleRequest(stream, user, request, url, response, params, yc, hasStartedStreaming)) { processed = true; break; } diff --git a/lib/remote/httphandler.hpp b/lib/remote/httphandler.hpp index 7e74329e0..cec5f58cd 100644 --- a/lib/remote/httphandler.hpp +++ b/lib/remote/httphandler.hpp @@ -8,7 +8,9 @@ #include "remote/httpresponse.hpp" #include "remote/apiuser.hpp" #include "base/registry.hpp" +#include "base/tlsstream.hpp" #include +#include #include namespace icinga @@ -25,18 +27,24 @@ public: DECLARE_PTR_TYPEDEFS(HttpHandler); virtual bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) = 0; static void Register(const Url::Ptr& url, const HttpHandler::Ptr& handler); static void ProcessRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, - boost::beast::http::response& response + boost::beast::http::response& response, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ); private: diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 87db3da5c..7ea206f5f 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -311,7 +311,7 @@ bool EnsureValidBody( } static inline -void ProcessRequest( +bool ProcessRequest( AsioTlsStream& stream, boost::beast::http::request& request, ApiUser::Ptr& authenticatedUser, @@ -321,11 +321,17 @@ void ProcessRequest( { namespace http = boost::beast::http; + bool hasStartedStreaming = false; + try { CpuBoundWork handlingRequest (yc); - HttpHandler::ProcessRequest(authenticatedUser, request, response); + HttpHandler::ProcessRequest(stream, authenticatedUser, request, response, yc, hasStartedStreaming); } catch (const std::exception& ex) { + if (hasStartedStreaming) { + return false; + } + http::response response; HttpUtility::SendJsonError(response, nullptr, 500, "Unhandled exception" , DiagnosticInformation(ex)); @@ -333,11 +339,17 @@ void ProcessRequest( http::async_write(stream, response, yc); stream.async_flush(yc); - return; + return true; + } + + if (hasStartedStreaming) { + return false; } http::async_write(stream, response, yc); stream.async_flush(yc); + + return true; } void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) @@ -419,7 +431,9 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) break; } - ProcessRequest(*m_Stream, request, authenticatedUser, response, yc); + if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, yc)) { + break; + } if (request.version() != 11 || request[http::field::connection] == "close") { break; diff --git a/lib/remote/infohandler.cpp b/lib/remote/infohandler.cpp index c605cca8b..38f99781d 100644 --- a/lib/remote/infohandler.cpp +++ b/lib/remote/infohandler.cpp @@ -9,11 +9,14 @@ using namespace icinga; REGISTER_URLHANDLER("/", InfoHandler); bool InfoHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/infohandler.hpp b/lib/remote/infohandler.hpp index 418bb9b4b..346340d81 100644 --- a/lib/remote/infohandler.hpp +++ b/lib/remote/infohandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(InfoHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/modifyobjecthandler.cpp b/lib/remote/modifyobjecthandler.cpp index af800cce1..2150903a1 100644 --- a/lib/remote/modifyobjecthandler.cpp +++ b/lib/remote/modifyobjecthandler.cpp @@ -13,11 +13,14 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", ModifyObjectHandler); bool ModifyObjectHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/modifyobjecthandler.hpp b/lib/remote/modifyobjecthandler.hpp index e63304beb..c9bb21612 100644 --- a/lib/remote/modifyobjecthandler.hpp +++ b/lib/remote/modifyobjecthandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ModifyObjectHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/objectqueryhandler.cpp b/lib/remote/objectqueryhandler.cpp index 63a67d853..2d9b79845 100644 --- a/lib/remote/objectqueryhandler.cpp +++ b/lib/remote/objectqueryhandler.cpp @@ -88,11 +88,14 @@ Dictionary::Ptr ObjectQueryHandler::SerializeObjectAttrs(const Object::Ptr& obje } bool ObjectQueryHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/objectqueryhandler.hpp b/lib/remote/objectqueryhandler.hpp index 86d3ef878..c35ff4066 100644 --- a/lib/remote/objectqueryhandler.hpp +++ b/lib/remote/objectqueryhandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(ObjectQueryHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; private: diff --git a/lib/remote/statushandler.cpp b/lib/remote/statushandler.cpp index 9b1175ba4..cc725e4f4 100644 --- a/lib/remote/statushandler.cpp +++ b/lib/remote/statushandler.cpp @@ -69,11 +69,14 @@ public: }; bool StatusHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/statushandler.hpp b/lib/remote/statushandler.hpp index a26161612..36f5efcf1 100644 --- a/lib/remote/statushandler.hpp +++ b/lib/remote/statushandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(StatusHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/templatequeryhandler.cpp b/lib/remote/templatequeryhandler.cpp index 9508163b1..38a225540 100644 --- a/lib/remote/templatequeryhandler.cpp +++ b/lib/remote/templatequeryhandler.cpp @@ -76,11 +76,14 @@ public: }; bool TemplateQueryHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/templatequeryhandler.hpp b/lib/remote/templatequeryhandler.hpp index e98bcd249..e9dec8896 100644 --- a/lib/remote/templatequeryhandler.hpp +++ b/lib/remote/templatequeryhandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(TemplateQueryHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/typequeryhandler.cpp b/lib/remote/typequeryhandler.cpp index 70c83f7e7..077113774 100644 --- a/lib/remote/typequeryhandler.cpp +++ b/lib/remote/typequeryhandler.cpp @@ -47,11 +47,14 @@ public: }; bool TypeQueryHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/typequeryhandler.hpp b/lib/remote/typequeryhandler.hpp index 69d65baec..84c04185d 100644 --- a/lib/remote/typequeryhandler.hpp +++ b/lib/remote/typequeryhandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(TypeQueryHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; diff --git a/lib/remote/variablequeryhandler.cpp b/lib/remote/variablequeryhandler.cpp index dcf01d3e9..05b503783 100644 --- a/lib/remote/variablequeryhandler.cpp +++ b/lib/remote/variablequeryhandler.cpp @@ -57,11 +57,14 @@ public: }; bool VariableQueryHandler::HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) { namespace http = boost::beast::http; diff --git a/lib/remote/variablequeryhandler.hpp b/lib/remote/variablequeryhandler.hpp index 5da2e13ab..ecc71a96a 100644 --- a/lib/remote/variablequeryhandler.hpp +++ b/lib/remote/variablequeryhandler.hpp @@ -14,11 +14,14 @@ public: DECLARE_PTR_TYPEDEFS(VariableQueryHandler); bool HandleRequest( + AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, const Url::Ptr& url, boost::beast::http::response& response, - const Dictionary::Ptr& params + const Dictionary::Ptr& params, + boost::asio::yield_context& yc, + bool& hasStartedStreaming ) override; }; From d7b465ce74235cd035eddb478863109c0d034cd1 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 12:25:25 +0100 Subject: [PATCH 17/67] Implement IoBoundWorkSlot --- lib/base/io-engine.cpp | 23 +++++++++++++++++++++++ lib/base/io-engine.hpp | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index 482d57176..daad42df0 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -64,6 +64,29 @@ void CpuBoundWork::Done() } } +IoBoundWorkSlot::IoBoundWorkSlot(boost::asio::yield_context yc) + : yc(yc) +{ + IoEngine::Get().m_CpuBoundSemaphore.fetch_add(1); +} + +IoBoundWorkSlot::~IoBoundWorkSlot() +{ + auto& ioEngine (IoEngine::Get()); + + for (;;) { + auto availableSlots (ioEngine.m_CpuBoundSemaphore.fetch_sub(1)); + + if (availableSlots < 1) { + ioEngine.m_CpuBoundSemaphore.fetch_add(1); + ioEngine.m_AlreadyExpiredTimer.async_wait(yc); + continue; + } + + break; + } +} + LazyInit> IoEngine::m_Instance ([]() { return std::unique_ptr(new IoEngine()); }); IoEngine& IoEngine::Get() diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index efeb56f99..ada80d1ca 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -55,6 +55,25 @@ private: bool m_Done; }; +/** + * Scope break for CPU-bound work done in an I/O thread + * + * @ingroup base + */ +class IoBoundWorkSlot +{ +public: + IoBoundWorkSlot(boost::asio::yield_context yc); + IoBoundWorkSlot(const IoBoundWorkSlot&) = delete; + IoBoundWorkSlot(IoBoundWorkSlot&&) = delete; + IoBoundWorkSlot& operator=(const IoBoundWorkSlot&) = delete; + IoBoundWorkSlot& operator=(IoBoundWorkSlot&&) = delete; + ~IoBoundWorkSlot(); + +private: + boost::asio::yield_context yc; +}; + /** * Async I/O engine * @@ -63,6 +82,7 @@ private: class IoEngine { friend CpuBoundWork; + friend IoBoundWorkSlot; public: IoEngine(const IoEngine&) = delete; From 7681ec10a4773ca392269922f4a20ea1775fec33 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 12:32:22 +0100 Subject: [PATCH 18/67] /v1/events: don't lock I/O thread --- lib/remote/eventqueue.cpp | 22 ++++++++++++++++++++++ lib/remote/eventqueue.hpp | 2 ++ lib/remote/eventshandler.cpp | 18 ++++++++++-------- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/lib/remote/eventqueue.cpp b/lib/remote/eventqueue.cpp index 20c4af688..6defc2114 100644 --- a/lib/remote/eventqueue.cpp +++ b/lib/remote/eventqueue.cpp @@ -2,8 +2,10 @@ #include "remote/eventqueue.hpp" #include "remote/filterutility.hpp" +#include "base/io-engine.hpp" #include "base/singleton.hpp" #include "base/logger.hpp" +#include using namespace icinga; @@ -100,6 +102,26 @@ Dictionary::Ptr EventQueue::WaitForEvent(void *client, double timeout) } } +Dictionary::Ptr EventQueue::WaitForEvent(void *client, boost::asio::yield_context yc) +{ + for (;;) { + { + boost::mutex::scoped_lock lock(m_Mutex); + + auto it = m_Events.find(client); + ASSERT(it != m_Events.end()); + + if (!it->second.empty()) { + Dictionary::Ptr result = *it->second.begin(); + it->second.pop_front(); + return result; + } + } + + IoBoundWorkSlot dontLockTheIoThreadWhileWaiting (yc); + } +} + std::vector EventQueue::GetQueuesForType(const String& type) { EventQueueRegistry::ItemMap queues = EventQueueRegistry::GetInstance()->GetItems(); diff --git a/lib/remote/eventqueue.hpp b/lib/remote/eventqueue.hpp index 45d23af23..8f6a76c0b 100644 --- a/lib/remote/eventqueue.hpp +++ b/lib/remote/eventqueue.hpp @@ -6,6 +6,7 @@ #include "remote/httphandler.hpp" #include "base/object.hpp" #include "config/expression.hpp" +#include #include #include #include @@ -31,6 +32,7 @@ public: void SetFilter(std::unique_ptr filter); Dictionary::Ptr WaitForEvent(void *client, double timeout = 5); + Dictionary::Ptr WaitForEvent(void *client, boost::asio::yield_context yc); static std::vector GetQueuesForType(const String& type); static void UnregisterIfUnused(const String& name, const EventQueue::Ptr& queue); diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index 99843d481..cf07d6305 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -6,6 +6,7 @@ #include "config/configcompiler.hpp" #include "config/expression.hpp" #include "base/defer.hpp" +#include "base/io-engine.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" #include @@ -91,23 +92,24 @@ bool EventsHandler::HandleRequest( response.result(http::status::ok); response.set(http::field::content_type, "application/json"); - http::async_write(stream, response, yc); - stream.async_flush(yc); + { + IoBoundWorkSlot dontLockTheIoThreadWhileWriting (yc); + + http::async_write(stream, response, yc); + stream.async_flush(yc); + } asio::const_buffer newLine ("\n", 1); for (;;) { - Dictionary::Ptr result = queue->WaitForEvent(&request); - - if (!result) - continue; - - String body = JsonEncode(result); + String body = JsonEncode(queue->WaitForEvent(&request, yc)); boost::algorithm::replace_all(body, "\n", ""); asio::const_buffer payload (body.CStr(), body.GetLength()); + IoBoundWorkSlot dontLockTheIoThreadWhileWriting (yc); + stream.async_write_some(payload, yc); stream.async_write_some(newLine, yc); stream.async_flush(yc); From ac72ca4ae6ee83bc561ca819e0aef7763c754e5d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 13:32:55 +0100 Subject: [PATCH 19/67] Don't warn that Boost.Coroutine v1 is deprecated --- CMakeLists.txt | 5 +++++ lib/base/io-engine.hpp | 7 ------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a3ee2e503..2b53844fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,11 @@ endif() find_package(Boost ${BOOST_MIN_VERSION} COMPONENTS context coroutine date_time thread system program_options regex REQUIRED) +# Boost.Coroutine2 (the successor of Boost.Coroutine) +# (1) doesn't even exist in old Boost versions and +# (2) isn't supported by ASIO, yet. +add_definitions(-DBOOST_COROUTINES_NO_DEPRECATION_WARNING) + link_directories(${Boost_LIBRARY_DIRS}) include_directories(${Boost_INCLUDE_DIRS}) diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index ada80d1ca..e383b2e42 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -20,13 +20,6 @@ #ifndef IO_ENGINE_H #define IO_ENGINE_H -/** - * Boost.Coroutine2 (the successor of Boost.Coroutine) - * (1) doesn't even exist in old Boost versions and - * (2) isn't supported by ASIO, yet. - */ -#define BOOST_COROUTINES_NO_DEPRECATION_WARNING 1 - #include "base/lazy-init.hpp" #include #include From 8c5d629d35e32ee0de779f481c47befd16f5ed84 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 14:36:23 +0100 Subject: [PATCH 20/67] /v1/events: don't truncate any events --- lib/remote/eventshandler.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index cf07d6305..e6f895cf3 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -10,6 +10,7 @@ #include "base/objectlock.hpp" #include "base/json.hpp" #include +#include #include using namespace icinga; @@ -110,8 +111,8 @@ bool EventsHandler::HandleRequest( IoBoundWorkSlot dontLockTheIoThreadWhileWriting (yc); - stream.async_write_some(payload, yc); - stream.async_write_some(newLine, yc); + asio::async_write(stream, payload, yc); + asio::async_write(stream, newLine, yc); stream.async_flush(yc); } } From 493a97f4f3caa43d1c86c4d6b0f67602fd1b7185 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 15:24:02 +0100 Subject: [PATCH 21/67] EnsureAcceptHeader(): fix wrong condition --- lib/remote/httpserverconnection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 7ea206f5f..a37b3a0b1 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -181,7 +181,7 @@ bool EnsureAcceptHeader( { namespace http = boost::beast::http; - if (request.method() == http::verb::get && request[http::field::accept] != "application/json") { + if (request.method() != http::verb::get && request[http::field::accept] != "application/json") { response.result(http::status::bad_request); response.set(http::field::content_type, "text/html"); response.body() = "

Accept header is missing or not set to 'application/json'.

"; From 282f8fd17397bb026de4845fed1de34c84716fac Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 15:43:58 +0100 Subject: [PATCH 22/67] IoEngine: explicitly join I/O threads --- lib/base/io-engine.cpp | 21 ++++++++++++++++++--- lib/base/io-engine.hpp | 9 +++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index daad42df0..0079ca251 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -99,7 +99,7 @@ boost::asio::io_service& IoEngine::GetIoService() return m_IoService; } -IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_AlreadyExpiredTimer(m_IoService) +IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_Threads(decltype(m_Threads)::size_type(std::thread::hardware_concurrency())), m_AlreadyExpiredTimer(m_IoService) { m_AlreadyExpiredTimer.expires_at(boost::posix_time::neg_infin); @@ -111,8 +111,21 @@ IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_AlreadyExpired m_CpuBoundSemaphore.store(concurrency - 1u); } - for (auto i (std::thread::hardware_concurrency()); i; --i) { - std::thread(&IoEngine::RunEventLoop, this).detach(); + for (auto& thread : m_Threads) { + thread = std::thread(&IoEngine::RunEventLoop, this); + } +} + +IoEngine::~IoEngine() +{ + for (auto& thread : m_Threads) { + m_IoService.post([]() { + throw TerminateIoThread(); + }); + } + + for (auto& thread : m_Threads) { + thread.join(); } } @@ -122,6 +135,8 @@ void IoEngine::RunEventLoop() try { m_IoService.run(); + break; + } catch (const TerminateIoThread&) { break; } catch (const std::exception& e) { Log(LogCritical, "IoEngine", "Exception during I/O operation!"); diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index e383b2e42..05610ca6f 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -22,7 +22,10 @@ #include "base/lazy-init.hpp" #include +#include #include +#include +#include #include #include #include @@ -82,6 +85,7 @@ public: IoEngine(IoEngine&&) = delete; IoEngine& operator=(const IoEngine&) = delete; IoEngine& operator=(IoEngine&&) = delete; + ~IoEngine(); static IoEngine& Get(); @@ -96,8 +100,13 @@ private: boost::asio::io_service m_IoService; boost::asio::io_service::work m_KeepAlive; + std::vector m_Threads; boost::asio::deadline_timer m_AlreadyExpiredTimer; std::atomic_uint_fast32_t m_CpuBoundSemaphore; }; +class TerminateIoThread : public std::exception +{ +}; + #endif /* IO_ENGINE_H */ From a6813ec786636957bd57648ee471362e68082c97 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 Feb 2019 18:19:56 +0100 Subject: [PATCH 23/67] ApiListener: restore previous bind(2) behavior --- lib/remote/apilistener.cpp | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 339ab83bf..6e9cc8b7b 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -362,12 +362,28 @@ bool ApiListener::AddListener(const String& node, const String& service) try { tcp::resolver resolver (io); tcp::resolver::query query (node, service, tcp::resolver::query::passive); - auto endpoint (resolver.resolve(query)->endpoint()); - acceptor->open(endpoint.protocol()); - acceptor->set_option(ip::v6_only(false)); - acceptor->set_option(tcp::acceptor::reuse_address(true)); - acceptor->bind(endpoint); + auto result (resolver.resolve(query)); + auto current (result.begin()); + + for (;;) { + try { + acceptor->open(current->endpoint().protocol()); + acceptor->set_option(ip::v6_only(false)); + acceptor->set_option(tcp::acceptor::reuse_address(true)); + acceptor->bind(current->endpoint()); + + break; + } catch (const std::exception&) { + if (++current == result.end()) { + throw; + } + + if (acceptor->is_open()) { + acceptor->close(); + } + } + } } catch (const std::exception&) { Log(LogCritical, "ApiListener") << "Cannot bind TCP socket for host '" << node << "' on port '" << service << "'."; From e9a64abd098390cb41f6024a33b9c5e14df5fe7a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 18 Feb 2019 14:23:59 +0100 Subject: [PATCH 24/67] ApiListener#ListenerCoroutineProc(): catch more edge cases --- lib/remote/apilistener.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 6e9cc8b7b..c0b75aecf 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -409,19 +409,17 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std namespace asio = boost::asio; auto& io (server->get_io_service()); - auto sslConn (std::make_shared(io, *sslContext)); for (;;) { try { + auto sslConn (std::make_shared(io, *sslContext)); + server->async_accept(sslConn->lowest_layer(), yc); + + asio::spawn(io, [this, sslConn](asio::yield_context yc) { NewClientHandler(yc, sslConn, String(), RoleServer); }); } catch (const std::exception& ex) { Log(LogCritical, "ApiListener") << "Cannot accept new connection: " << DiagnosticInformation(ex, false); - continue; } - - asio::spawn(io, [this, sslConn](asio::yield_context yc) { NewClientHandler(yc, sslConn, String(), RoleServer); }); - - sslConn = std::make_shared(io, *sslContext); } } From 832365195df7e9d29eec486d1235897d9b137ca2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 18 Feb 2019 14:56:45 +0100 Subject: [PATCH 25/67] ApiListener: connect(2) via Boost ASIO --- lib/remote/apilistener.cpp | 239 ++++++++----------------------------- lib/remote/apilistener.hpp | 3 - 2 files changed, 49 insertions(+), 193 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index c0b75aecf..b5fc8e18b 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -430,210 +430,70 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std */ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint) { - { - ObjectLock olock(this); + namespace asio = boost::asio; + using asio::ip::tcp; - auto sslContext (m_SSLContext); + auto sslContext (m_SSLContext); - if (!sslContext) { - Log(LogCritical, "ApiListener", "SSL context is required for AddConnection()"); - return; - } - } - - String host = endpoint->GetHost(); - String port = endpoint->GetPort(); - - Log(LogInformation, "ApiListener") - << "Reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'"; - - TcpSocket::Ptr client = new TcpSocket(); - - try { - client->Connect(host, port); - - NewClientHandler(client, endpoint->GetName(), RoleClient); - - endpoint->SetConnecting(false); - Log(LogInformation, "ApiListener") - << "Finished reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'"; - } catch (const std::exception& ex) { - endpoint->SetConnecting(false); - client->Close(); - - std::ostringstream info; - info << "Cannot connect to host '" << host << "' on port '" << port << "'"; - Log(LogCritical, "ApiListener", info.str()); - Log(LogDebug, "ApiListener") - << info.str() << "\n" << DiagnosticInformation(ex); - } -} - -void ApiListener::NewClientHandler(const Socket::Ptr& client, const String& hostname, ConnectionRole role) -{ - try { - NewClientHandlerInternal(client, hostname, role); - } catch (const std::exception& ex) { - Log(LogCritical, "ApiListener") - << "Exception while handling new API client connection: " << DiagnosticInformation(ex, false); - - Log(LogDebug, "ApiListener") - << "Exception while handling new API client connection: " << DiagnosticInformation(ex); - } -} - -/** - * Processes a new client connection. - * - * @param client The new client. - */ -void ApiListener::NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role) -{ - CONTEXT("Handling new API client connection"); - - String conninfo; - - if (role == RoleClient) - conninfo = "to"; - else - conninfo = "from"; - - conninfo += " " + client->GetPeerAddress(); - - TlsStream::Ptr tlsStream; - - String environmentName = Application::GetAppEnvironment(); - - String serverName = hostname; - - if (!environmentName.IsEmpty()) - serverName += ":" + environmentName; - - { - ObjectLock olock(this); - try { - tlsStream = new TlsStream(client, serverName, role, m_SSLContext); - } catch (const std::exception&) { - Log(LogCritical, "ApiListener") - << "Cannot create TLS stream from client connection (" << conninfo << ")"; - return; - } - } - - try { - tlsStream->Handshake(); - } catch (const std::exception& ex) { - Log(LogCritical, "ApiListener") - << "Client TLS handshake failed (" << conninfo << "): " << DiagnosticInformation(ex, false); - tlsStream->Close(); + if (!sslContext) { + Log(LogCritical, "ApiListener", "SSL context is required for AddConnection()"); return; } - std::shared_ptr cert = tlsStream->GetPeerCertificate(); - String identity; - Endpoint::Ptr endpoint; - bool verify_ok = false; + auto& io (IoEngine::Get().GetIoService()); - if (cert) { - try { - identity = GetCertificateCN(cert); - } catch (const std::exception&) { - Log(LogCritical, "ApiListener") - << "Cannot get certificate common name from cert path: '" << GetDefaultCertPath() << "'."; - tlsStream->Close(); - return; - } + asio::spawn(io, [this, endpoint, &io, sslContext](asio::yield_context yc) { + String host = endpoint->GetHost(); + String port = endpoint->GetPort(); - verify_ok = tlsStream->IsVerifyOK(); - if (!hostname.IsEmpty()) { - if (identity != hostname) { - Log(LogWarning, "ApiListener") - << "Unexpected certificate common name while connecting to endpoint '" - << hostname << "': got '" << identity << "'"; - tlsStream->Close(); - return; - } else if (!verify_ok) { - Log(LogWarning, "ApiListener") - << "Certificate validation failed for endpoint '" << hostname - << "': " << tlsStream->GetVerifyError(); - } - } - - if (verify_ok) - endpoint = Endpoint::GetByName(identity); - - { - Log log(LogInformation, "ApiListener"); - - log << "New client connection for identity '" << identity << "' " << conninfo; - - if (!verify_ok) - log << " (certificate validation failed: " << tlsStream->GetVerifyError() << ")"; - else if (!endpoint) - log << " (no Endpoint object found for identity)"; - } - } else { Log(LogInformation, "ApiListener") - << "New client connection " << conninfo << " (no client certificate)"; - } + << "Reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'"; - ClientType ctype; + try { + auto sslConn (std::make_shared(io, *sslContext)); - if (role == RoleClient) { - Dictionary::Ptr message = new Dictionary({ - { "jsonrpc", "2.0" }, - { "method", "icinga::Hello" }, - { "params", new Dictionary() } - }); + { + tcp::resolver resolver (io); + tcp::resolver::query query (host, port); + auto result (resolver.async_resolve(query, yc)); + auto current (result.begin()); - JsonRpc::SendMessage(tlsStream, message); - ctype = ClientJsonRpc; - } else { - tlsStream->WaitForData(10); + for (;;) { + auto& tcpConn (sslConn->lowest_layer()); - if (!tlsStream->IsDataAvailable()) { - if (identity.IsEmpty()) - Log(LogInformation, "ApiListener") - << "No data received on new API connection. " - << "Ensure that the remote endpoints are properly configured in a cluster setup."; - else - Log(LogWarning, "ApiListener") - << "No data received on new API connection for identity '" << identity << "'. " - << "Ensure that the remote endpoints are properly configured in a cluster setup."; - tlsStream->Close(); - return; - } + try { + tcpConn.open(current->endpoint().protocol()); + tcpConn.set_option(tcp::socket::keep_alive(true)); + tcpConn.async_connect(current->endpoint(), yc); - char firstByte; - tlsStream->Peek(&firstByte, 1, false); + break; + } catch (const std::exception&) { + if (++current == result.end()) { + throw; + } - if (firstByte >= '0' && firstByte <= '9') - ctype = ClientJsonRpc; - else - ctype = ClientHttp; - } - - if (ctype == ClientJsonRpc) { - Log(LogNotice, "ApiListener", "New JSON-RPC client"); - - JsonRpcConnection::Ptr aclient = new JsonRpcConnection(identity, verify_ok, tlsStream, role); - aclient->Start(); - - if (endpoint) { - bool needSync = !endpoint->GetConnected(); - - endpoint->AddClient(aclient); - - m_SyncQueue.Enqueue(std::bind(&ApiListener::SyncClient, this, aclient, endpoint, needSync)); - } else { - if (!AddAnonymousClient(aclient)) { - Log(LogNotice, "ApiListener") - << "Ignoring anonymous JSON-RPC connection " << conninfo - << ". Max connections (" << GetMaxAnonymousClients() << ") exceeded."; - aclient->Disconnect(); + if (tcpConn.is_open()) { + tcpConn.close(); + } + } + } } + + NewClientHandler(yc, sslConn, endpoint->GetName(), RoleClient); + + endpoint->SetConnecting(false); + Log(LogInformation, "ApiListener") + << "Finished reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'"; + } catch (const std::exception& ex) { + endpoint->SetConnecting(false); + + std::ostringstream info; + info << "Cannot connect to host '" << host << "' on port '" << port << "'"; + Log(LogCritical, "ApiListener", info.str()); + Log(LogDebug, "ApiListener") + << info.str() << "\n" << DiagnosticInformation(ex); } - } + }); } void ApiListener::NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role) @@ -1004,8 +864,7 @@ void ApiListener::ApiReconnectTimerHandler() /* Set connecting state to prevent duplicated queue inserts later. */ endpoint->SetConnecting(true); - /* Use dynamic thread pool with additional on demand resources with fast throughput. */ - EnqueueAsyncCallback(std::bind(&ApiListener::AddConnection, this, endpoint), LowLatencyScheduler); + AddConnection(endpoint); } } diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 093837493..b88f1de1e 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -130,9 +130,6 @@ private: bool AddListener(const String& node, const String& service); void AddConnection(const Endpoint::Ptr& endpoint); - void NewClientHandler(const Socket::Ptr& client, const String& hostname, ConnectionRole role); - void NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role); - void NewClientHandler(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role); void NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role); void ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext); From 43658de5294202b0990bfeea74865535c187ed97 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 18 Feb 2019 15:21:50 +0100 Subject: [PATCH 26/67] NetString::WriteStringToStream(): add Boost ASIO overload --- lib/base/netstring.cpp | 28 ++++++++++++++++++++++++++++ lib/base/netstring.hpp | 4 ++++ 2 files changed, 32 insertions(+) diff --git a/lib/base/netstring.cpp b/lib/base/netstring.cpp index 7fad3453c..debed2bda 100644 --- a/lib/base/netstring.cpp +++ b/lib/base/netstring.cpp @@ -2,7 +2,12 @@ #include "base/netstring.hpp" #include "base/debug.hpp" +#include "base/tlsstream.hpp" +#include #include +#include +#include +#include using namespace icinga; @@ -110,6 +115,29 @@ size_t NetString::WriteStringToStream(const Stream::Ptr& stream, const String& s return msg.GetLength(); } +/** + * Writes data into a stream using the netstring format and returns bytes written. + * + * @param stream The stream. + * @param str The String that is to be written. + * + * @return The amount of bytes written. + */ +size_t NetString::WriteStringToStream(const std::shared_ptr& stream, const String& str, boost::asio::yield_context yc) +{ + namespace asio = boost::asio; + + std::ostringstream msgbuf; + WriteStringToStream(msgbuf, str); + + String msg = msgbuf.str(); + asio::const_buffer msgBuf (msg.CStr(), msg.GetLength()); + + asio::async_write(*stream, msgBuf, yc); + + return msg.GetLength(); +} + /** * Writes data into a stream using the netstring format. * diff --git a/lib/base/netstring.hpp b/lib/base/netstring.hpp index 10ff4e2fc..f54d70c17 100644 --- a/lib/base/netstring.hpp +++ b/lib/base/netstring.hpp @@ -5,6 +5,9 @@ #include "base/i2-base.hpp" #include "base/stream.hpp" +#include "base/tlsstream.hpp" +#include +#include namespace icinga { @@ -24,6 +27,7 @@ public: static StreamReadStatus ReadStringFromStream(const Stream::Ptr& stream, String *message, StreamReadContext& context, bool may_wait = false, ssize_t maxMessageLength = -1); static size_t WriteStringToStream(const Stream::Ptr& stream, const String& message); + static size_t WriteStringToStream(const std::shared_ptr& stream, const String& message, boost::asio::yield_context yc); static void WriteStringToStream(std::ostream& stream, const String& message); private: From 49ac7777e02aa9dd1c465c569aa4652e4fcfe180 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 18 Feb 2019 15:23:41 +0100 Subject: [PATCH 27/67] JsonRpc::SendMessage(): add Boost ASIO overload --- lib/remote/jsonrpc.cpp | 22 ++++++++++++++++++++++ lib/remote/jsonrpc.hpp | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/lib/remote/jsonrpc.cpp b/lib/remote/jsonrpc.cpp index 1c429ea7c..b07d0e6ed 100644 --- a/lib/remote/jsonrpc.cpp +++ b/lib/remote/jsonrpc.cpp @@ -6,7 +6,10 @@ #include "base/console.hpp" #include "base/scriptglobal.hpp" #include "base/convert.hpp" +#include "base/tlsstream.hpp" #include +#include +#include using namespace icinga; @@ -55,6 +58,25 @@ size_t JsonRpc::SendMessage(const Stream::Ptr& stream, const Dictionary::Ptr& me return NetString::WriteStringToStream(stream, json); } +/** + * Sends a message to the connected peer and returns the bytes sent. + * + * @param message The message. + * + * @return The amount of bytes sent. + */ +size_t JsonRpc::SendMessage(const std::shared_ptr& stream, const Dictionary::Ptr& message, boost::asio::yield_context yc) +{ + String json = JsonEncode(message); + +#ifdef I2_DEBUG + if (GetDebugJsonRpcCached()) + std::cerr << ConsoleColorTag(Console_ForegroundBlue) << ">> " << json << ConsoleColorTag(Console_Normal) << "\n"; +#endif /* I2_DEBUG */ + + return NetString::WriteStringToStream(stream, json, yc); +} + StreamReadStatus JsonRpc::ReadMessage(const Stream::Ptr& stream, String *message, StreamReadContext& src, bool may_wait, ssize_t maxMessageLength) { String jsonString; diff --git a/lib/remote/jsonrpc.hpp b/lib/remote/jsonrpc.hpp index 44e8a9360..2efe8022b 100644 --- a/lib/remote/jsonrpc.hpp +++ b/lib/remote/jsonrpc.hpp @@ -5,7 +5,10 @@ #include "base/stream.hpp" #include "base/dictionary.hpp" +#include "base/tlsstream.hpp" #include "remote/i2-remote.hpp" +#include +#include namespace icinga { @@ -19,6 +22,7 @@ class JsonRpc { public: static size_t SendMessage(const Stream::Ptr& stream, const Dictionary::Ptr& message); + static size_t SendMessage(const std::shared_ptr& stream, const Dictionary::Ptr& message, boost::asio::yield_context yc); static StreamReadStatus ReadMessage(const Stream::Ptr& stream, String *message, StreamReadContext& src, bool may_wait = false, ssize_t maxMessageLength = -1); static Dictionary::Ptr DecodeMessage(const String& message); From 48b5824e37ee188bcef2d0f686445e5061f782c6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 18 Feb 2019 15:31:58 +0100 Subject: [PATCH 28/67] ApiListener: send icinga::Hello message --- lib/remote/apilistener.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index b5fc8e18b..e0225aa10 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -626,7 +626,17 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const ClientType ctype; - if (role != RoleClient) { + if (role == RoleClient) { + JsonRpc::SendMessage(client, new Dictionary({ + { "jsonrpc", "2.0" }, + { "method", "icinga::Hello" }, + { "params", new Dictionary() } + }), yc); + + client->async_flush(yc); + + ctype = ClientJsonRpc; + } else { { boost::system::error_code ec; From b26808414c9cdc5d8d9243771fd8fca4c76128e4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 11:20:39 +0100 Subject: [PATCH 29/67] NetString::ReadStringFromStream(): add Boost ASIO overload --- lib/base/netstring.cpp | 82 ++++++++++++++++++++++++++++++++++++++++++ lib/base/netstring.hpp | 2 ++ 2 files changed, 84 insertions(+) diff --git a/lib/base/netstring.cpp b/lib/base/netstring.cpp index debed2bda..489a8b40d 100644 --- a/lib/base/netstring.cpp +++ b/lib/base/netstring.cpp @@ -3,9 +3,12 @@ #include "base/netstring.hpp" #include "base/debug.hpp" #include "base/tlsstream.hpp" +#include #include #include +#include #include +#include #include #include @@ -115,6 +118,85 @@ size_t NetString::WriteStringToStream(const Stream::Ptr& stream, const String& s return msg.GetLength(); } +/** + * Reads data from a stream in netstring format. + * + * @param stream The stream to read from. + * @returns The String that has been read from the IOQueue. + * @exception invalid_argument The input stream is invalid. + * @see https://github.com/PeterScott/netstring-c/blob/master/netstring.c + */ +String NetString::ReadStringFromStream(const std::shared_ptr& stream, + boost::asio::yield_context yc, ssize_t maxMessageLength) +{ + namespace asio = boost::asio; + + size_t len = 0; + bool leadingZero = false; + + for (uint_fast8_t readBytes = 0;; ++readBytes) { + char byte = 0; + + { + asio::mutable_buffer byteBuf (&byte, 1); + asio::async_read(*stream, byteBuf, yc); + } + + if (isdigit(byte)) { + if (readBytes == 9) { + BOOST_THROW_EXCEPTION(std::invalid_argument("Length specifier must not exceed 9 characters")); + } + + if (leadingZero) { + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid NetString (leading zero)")); + } + + len = len * 10u + size_t(byte - '0'); + + if (!readBytes && byte == '0') { + leadingZero = true; + } + } else if (byte == ':') { + if (!readBytes) { + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid NetString (no length specifier)")); + } + + break; + } else { + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid NetString (missing :)")); + } + } + + if (maxMessageLength >= 0 && len > maxMessageLength) { + std::stringstream errorMessage; + errorMessage << "Max data length exceeded: " << (maxMessageLength / 1024) << " KB"; + + BOOST_THROW_EXCEPTION(std::invalid_argument(errorMessage.str())); + } + + String payload; + + if (len) { + payload.Append(len, 0); + + asio::mutable_buffer payloadBuf (&*payload.Begin(), payload.GetLength()); + asio::async_read(*stream, payloadBuf, yc); + } + + char trailer = 0; + + { + asio::mutable_buffer trailerBuf (&trailer, 1); + asio::async_read(*stream, trailerBuf, yc); + } + + if (trailer != ',') { + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid NetString (missing ,)")); + } + + return std::move(payload); +} + /** * Writes data into a stream using the netstring format and returns bytes written. * diff --git a/lib/base/netstring.hpp b/lib/base/netstring.hpp index f54d70c17..f84eac7a3 100644 --- a/lib/base/netstring.hpp +++ b/lib/base/netstring.hpp @@ -26,6 +26,8 @@ class NetString public: static StreamReadStatus ReadStringFromStream(const Stream::Ptr& stream, String *message, StreamReadContext& context, bool may_wait = false, ssize_t maxMessageLength = -1); + static String ReadStringFromStream(const std::shared_ptr& stream, + boost::asio::yield_context yc, ssize_t maxMessageLength = -1); static size_t WriteStringToStream(const Stream::Ptr& stream, const String& message); static size_t WriteStringToStream(const std::shared_ptr& stream, const String& message, boost::asio::yield_context yc); static void WriteStringToStream(std::ostream& stream, const String& message); From c76947e8b932791d0fddbcee9765774aea02167c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 11:29:45 +0100 Subject: [PATCH 30/67] JsonRpc::ReadMessage(): add Boost ASIO overload --- lib/remote/jsonrpc.cpp | 13 +++++++++++++ lib/remote/jsonrpc.hpp | 1 + 2 files changed, 14 insertions(+) diff --git a/lib/remote/jsonrpc.cpp b/lib/remote/jsonrpc.cpp index b07d0e6ed..5f4b6ae91 100644 --- a/lib/remote/jsonrpc.cpp +++ b/lib/remote/jsonrpc.cpp @@ -9,6 +9,7 @@ #include "base/tlsstream.hpp" #include #include +#include #include using namespace icinga; @@ -95,6 +96,18 @@ StreamReadStatus JsonRpc::ReadMessage(const Stream::Ptr& stream, String *message return StatusNewItem; } +String JsonRpc::ReadMessage(const std::shared_ptr& stream, boost::asio::yield_context yc, ssize_t maxMessageLength) +{ + String jsonString = NetString::ReadStringFromStream(stream, yc, maxMessageLength); + +#ifdef I2_DEBUG + if (GetDebugJsonRpcCached()) + std::cerr << ConsoleColorTag(Console_ForegroundBlue) << "<< " << jsonString << ConsoleColorTag(Console_Normal) << "\n"; +#endif /* I2_DEBUG */ + + return std::move(jsonString); +} + Dictionary::Ptr JsonRpc::DecodeMessage(const String& message) { Value value = JsonDecode(message); diff --git a/lib/remote/jsonrpc.hpp b/lib/remote/jsonrpc.hpp index 2efe8022b..137d42a3b 100644 --- a/lib/remote/jsonrpc.hpp +++ b/lib/remote/jsonrpc.hpp @@ -24,6 +24,7 @@ public: static size_t SendMessage(const Stream::Ptr& stream, const Dictionary::Ptr& message); static size_t SendMessage(const std::shared_ptr& stream, const Dictionary::Ptr& message, boost::asio::yield_context yc); static StreamReadStatus ReadMessage(const Stream::Ptr& stream, String *message, StreamReadContext& src, bool may_wait = false, ssize_t maxMessageLength = -1); + static String ReadMessage(const std::shared_ptr& stream, boost::asio::yield_context yc, ssize_t maxMessageLength = -1); static Dictionary::Ptr DecodeMessage(const String& message); private: From 6c86c127f1a3bf6f33d6b59e306d48e6f4d4d3b2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 13:57:36 +0100 Subject: [PATCH 31/67] Port JsonRpcConnection to Boost ASIO --- lib/remote/apilistener-configsync.cpp | 4 +- lib/remote/apilistener.cpp | 58 ++-- lib/remote/jsonrpcconnection-heartbeat.cpp | 33 --- lib/remote/jsonrpcconnection-pki.cpp | 16 +- lib/remote/jsonrpcconnection.cpp | 317 ++++++++------------- lib/remote/jsonrpcconnection.hpp | 38 ++- 6 files changed, 179 insertions(+), 287 deletions(-) diff --git a/lib/remote/apilistener-configsync.cpp b/lib/remote/apilistener-configsync.cpp index d4caff91a..e06d1d887 100644 --- a/lib/remote/apilistener-configsync.cpp +++ b/lib/remote/apilistener-configsync.cpp @@ -323,7 +323,7 @@ void ApiListener::UpdateConfigObject(const ConfigObject::Ptr& object, const Mess #endif /* I2_DEBUG */ if (client) - JsonRpc::SendMessage(client->GetStream(), message); + client->SendMessage(message); else { Zone::Ptr target = static_pointer_cast(object->GetZone()); @@ -373,7 +373,7 @@ void ApiListener::DeleteConfigObject(const ConfigObject::Ptr& object, const Mess #endif /* I2_DEBUG */ if (client) - JsonRpc::SendMessage(client->GetStream(), message); + client->SendMessage(message); else { Zone::Ptr target = static_pointer_cast(object->GetZone()); diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index e0225aa10..87933ae7a 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -212,16 +212,6 @@ void ApiListener::UpdateSSLContext() } m_SSLContext = context; - - for (const Endpoint::Ptr& endpoint : ConfigType::GetObjectsByType()) { - for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { - client->Disconnect(); - } - } - - for (const JsonRpcConnection::Ptr& client : m_AnonymousClients) { - client->Disconnect(); - } } void ApiListener::OnAllConfigLoaded() @@ -669,7 +659,33 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const } } - if (ctype != ClientJsonRpc) { + if (ctype == ClientJsonRpc) { + Log(LogNotice, "ApiListener", "New JSON-RPC client"); + + JsonRpcConnection::Ptr aclient = new JsonRpcConnection(identity, verify_ok, client, role); + + if (endpoint) { + bool needSync = !endpoint->GetConnected(); + + endpoint->AddClient(aclient); + + asio::spawn(client->get_io_service(), [this, aclient, endpoint, needSync](asio::yield_context yc) { + CpuBoundWork syncClient (yc); + + SyncClient(aclient, endpoint, needSync); + }); + } else if (!AddAnonymousClient(aclient)) { + Log(LogNotice, "ApiListener") + << "Ignoring anonymous JSON-RPC connection " << conninfo + << ". Max connections (" << GetMaxAnonymousClients() << ") exceeded."; + + aclient = nullptr; + } + + if (aclient) { + aclient->Start(); + } + } else { Log(LogNotice, "ApiListener", "New HTTP client"); HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, client); @@ -810,10 +826,9 @@ void ApiListener::ApiTimerHandler() } for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { - if (client->GetTimestamp() != maxTs) - client->Disconnect(); - else + if (client->GetTimestamp() == maxTs) { client->SendMessage(lmessage); + } } Log(LogNotice, "ApiListener") @@ -1280,8 +1295,7 @@ void ApiListener::ReplayLog(const JsonRpcConnection::Ptr& client) } try { - size_t bytesSent = NetString::WriteStringToStream(client->GetStream(), pmessage->Get("message")); - endpoint->AddMessageSent(bytesSent); + client->SendMessage(JsonDecode(pmessage->Get("message"))); count++; } catch (const std::exception& ex) { Log(LogWarning, "ApiListener") @@ -1306,8 +1320,7 @@ void ApiListener::ReplayLog(const JsonRpcConnection::Ptr& client) }) } }); - size_t bytesSent = JsonRpc::SendMessage(client->GetStream(), lmessage); - endpoint->AddMessageSent(bytesSent); + client->SendMessage(lmessage); } } @@ -1426,11 +1439,8 @@ std::pair ApiListener::GetStatus() /* connection stats */ size_t jsonRpcAnonymousClients = GetAnonymousClients().size(); size_t httpClients = GetHttpClients().size(); - size_t workQueueItems = JsonRpcConnection::GetWorkQueueLength(); - size_t workQueueCount = JsonRpcConnection::GetWorkQueueCount(); size_t syncQueueItems = m_SyncQueue.GetLength(); size_t relayQueueItems = m_RelayQueue.GetLength(); - double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate(); double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0; double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0; @@ -1446,11 +1456,8 @@ std::pair ApiListener::GetStatus() { "json_rpc", new Dictionary({ { "anonymous_clients", jsonRpcAnonymousClients }, - { "work_queue_items", workQueueItems }, - { "work_queue_count", workQueueCount }, { "sync_queue_items", syncQueueItems }, { "relay_queue_items", relayQueueItems }, - { "work_queue_item_rate", workQueueItemRate }, { "sync_queue_item_rate", syncQueueItemRate }, { "relay_queue_item_rate", relayQueueItemRate } }) }, @@ -1467,12 +1474,9 @@ std::pair ApiListener::GetStatus() perfdata->Set("num_json_rpc_anonymous_clients", jsonRpcAnonymousClients); perfdata->Set("num_http_clients", httpClients); - perfdata->Set("num_json_rpc_work_queue_items", workQueueItems); - perfdata->Set("num_json_rpc_work_queue_count", workQueueCount); perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems); perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems); - perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate); perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate); perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate); diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 5b466830c..91fa51891 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -12,41 +12,8 @@ using namespace icinga; REGISTER_APIFUNCTION(Heartbeat, event, &JsonRpcConnection::HeartbeatAPIHandler); -void JsonRpcConnection::HeartbeatTimerHandler() -{ - for (const Endpoint::Ptr& endpoint : ConfigType::GetObjectsByType()) { - for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { - if (client->m_NextHeartbeat != 0 && client->m_NextHeartbeat < Utility::GetTime()) { - Log(LogWarning, "JsonRpcConnection") - << "Client for endpoint '" << endpoint->GetName() << "' has requested " - << "heartbeat message but hasn't responded in time. Closing connection."; - - client->Disconnect(); - continue; - } - - Dictionary::Ptr request = new Dictionary({ - { "jsonrpc", "2.0" }, - { "method", "event::Heartbeat" }, - { "params", new Dictionary({ - { "timeout", 120 } - }) } - }); - - client->SendMessage(request); - } - } -} - Value JsonRpcConnection::HeartbeatAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { - Value vtimeout = params->Get("timeout"); - - if (!vtimeout.IsEmpty()) { - origin->FromClient->m_NextHeartbeat = Utility::GetTime() + vtimeout; - origin->FromClient->m_HeartbeatTimeout = vtimeout; - } - return Empty; } diff --git a/lib/remote/jsonrpcconnection-pki.cpp b/lib/remote/jsonrpcconnection-pki.cpp index 8eb82ed40..66f88479b 100644 --- a/lib/remote/jsonrpcconnection-pki.cpp +++ b/lib/remote/jsonrpcconnection-pki.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include using namespace icinga; @@ -30,10 +32,12 @@ Value RequestCertificateHandler(const MessageOrigin::Ptr& origin, const Dictiona Dictionary::Ptr result = new Dictionary(); /* Use the presented client certificate if not provided. */ - if (certText.IsEmpty()) - cert = origin->FromClient->GetStream()->GetPeerCertificate(); - else + if (certText.IsEmpty()) { + auto stream (origin->FromClient->GetStream()); + cert = std::shared_ptr(SSL_get_peer_certificate(stream->next_layer().native_handle()), X509_free); + } else { cert = StringToCertificate(certText); + } if (!cert) { Log(LogWarning, "JsonRpcConnection") << "No certificate or CSR received"; @@ -121,7 +125,7 @@ Value RequestCertificateHandler(const MessageOrigin::Ptr& origin, const Dictiona { "method", "pki::UpdateCertificate" }, { "params", result } }); - JsonRpc::SendMessage(client->GetStream(), message); + client->SendMessage(message); return result; } @@ -192,7 +196,7 @@ Value RequestCertificateHandler(const MessageOrigin::Ptr& origin, const Dictiona { "method", "pki::UpdateCertificate" }, { "params", result } }); - JsonRpc::SendMessage(client->GetStream(), message); + client->SendMessage(message); return result; @@ -255,7 +259,7 @@ void JsonRpcConnection::SendCertificateRequest(const JsonRpcConnection::Ptr& acl * or b) the local zone and all parents. */ if (aclient) - JsonRpc::SendMessage(aclient->GetStream(), message); + aclient->SendMessage(message); else listener->RelayMessage(origin, Zone::GetLocalZone(), message, false); } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 99d0d7feb..3840a8c89 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -5,11 +5,17 @@ #include "remote/apifunction.hpp" #include "remote/jsonrpc.hpp" #include "base/configtype.hpp" +#include "base/io-engine.hpp" #include "base/objectlock.hpp" #include "base/utility.hpp" #include "base/logger.hpp" #include "base/exception.hpp" #include "base/convert.hpp" +#include "base/tlsstream.hpp" +#include +#include +#include +#include #include using namespace icinga; @@ -17,50 +23,121 @@ using namespace icinga; static Value SetLogPositionHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); REGISTER_APIFUNCTION(SetLogPosition, log, &SetLogPositionHandler); -static boost::once_flag l_JsonRpcConnectionOnceFlag = BOOST_ONCE_INIT; -static Timer::Ptr l_JsonRpcConnectionTimeoutTimer; -static WorkQueue *l_JsonRpcConnectionWorkQueues; -static size_t l_JsonRpcConnectionWorkQueueCount; -static int l_JsonRpcConnectionNextID; -static Timer::Ptr l_HeartbeatTimer; - JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, - TlsStream::Ptr stream, ConnectionRole role) - : m_ID(l_JsonRpcConnectionNextID++), m_Identity(identity), m_Authenticated(authenticated), m_Stream(std::move(stream)), - m_Role(role), m_Timestamp(Utility::GetTime()), m_Seen(Utility::GetTime()), m_NextHeartbeat(0), m_HeartbeatTimeout(0) + const std::shared_ptr& stream, ConnectionRole role) + : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), + m_Role(role), m_Timestamp(Utility::GetTime()), m_IoStrand(stream->get_io_service()), + m_OutgoingMessagesQueued(stream->get_io_service()), m_ReaderHasError(false), m_RunningCoroutines(0) { - boost::call_once(l_JsonRpcConnectionOnceFlag, &JsonRpcConnection::StaticInitialize); - if (authenticated) m_Endpoint = Endpoint::GetByName(identity); -} -void JsonRpcConnection::StaticInitialize() -{ - l_JsonRpcConnectionTimeoutTimer = new Timer(); - l_JsonRpcConnectionTimeoutTimer->OnTimerExpired.connect(std::bind(&JsonRpcConnection::TimeoutTimerHandler)); - l_JsonRpcConnectionTimeoutTimer->SetInterval(15); - l_JsonRpcConnectionTimeoutTimer->Start(); - - l_JsonRpcConnectionWorkQueueCount = Configuration::Concurrency; - l_JsonRpcConnectionWorkQueues = new WorkQueue[l_JsonRpcConnectionWorkQueueCount]; - - for (size_t i = 0; i < l_JsonRpcConnectionWorkQueueCount; i++) { - l_JsonRpcConnectionWorkQueues[i].SetName("JsonRpcConnection, #" + Convert::ToString(i)); - } - - l_HeartbeatTimer = new Timer(); - l_HeartbeatTimer->OnTimerExpired.connect(std::bind(&JsonRpcConnection::HeartbeatTimerHandler)); - l_HeartbeatTimer->SetInterval(10); - l_HeartbeatTimer->Start(); + m_OutgoingMessagesQueued.expires_at(boost::posix_time::pos_infin); } void JsonRpcConnection::Start() { - /* the stream holds an owning reference to this object through the callback we're registering here */ - m_Stream->RegisterDataHandler(std::bind(&JsonRpcConnection::DataAvailableHandler, JsonRpcConnection::Ptr(this))); - if (m_Stream->IsDataAvailable()) - DataAvailableHandler(); + namespace asio = boost::asio; + + m_RunningCoroutines = 2; + + asio::spawn(m_IoStrand, [this](asio::yield_context yc) { HandleIncomingMessages(yc); }); + asio::spawn(m_IoStrand, [this](asio::yield_context yc) { WriteOutgoingMessages(yc); }); +} + +void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) +{ + Defer shutdownStreamOnce ([this, &yc]() { + m_ReaderHasError = true; + m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + + ShutdownStreamOnce(yc); + }); + + for (;;) { + String message; + + try { + message = JsonRpc::ReadMessage(m_Stream, yc, m_Endpoint ? -1 : 1024 * 1024); + } catch (const std::exception& ex) { + Log(LogWarning, "JsonRpcConnection") + << "Error while reading JSON-RPC message for identity '" << m_Identity + << "': " << DiagnosticInformation(ex); + + break; + } + + try { + CpuBoundWork handleMessage (yc); + + MessageHandler(message); + } catch (const std::exception& ex) { + Log(LogWarning, "JsonRpcConnection") + << "Error while processing JSON-RPC message for identity '" << m_Identity + << "': " << DiagnosticInformation(ex); + + break; + } + } +} + +void JsonRpcConnection::WriteOutgoingMessages(boost::asio::yield_context yc) +{ + Defer shutdownStreamOnce ([this, &yc]() { ShutdownStreamOnce(yc); }); + + do { + try { + m_OutgoingMessagesQueued.async_wait(yc); + } catch (...) { + } + + auto queue (std::move(m_OutgoingMessagesQueue)); + + m_OutgoingMessagesQueue.clear(); + m_OutgoingMessagesQueued.expires_at(boost::posix_time::pos_infin); + + if (!queue.empty()) { + try { + for (auto& message : queue) { + size_t bytesSent = JsonRpc::SendMessage(m_Stream, message, yc); + + if (m_Endpoint) { + m_Endpoint->AddMessageSent(bytesSent); + } + } + + m_Stream->async_flush(yc); + } catch (const std::exception& ex) { + std::ostringstream info; + info << "Error while sending JSON-RPC message for identity '" << m_Identity << "'"; + Log(LogWarning, "JsonRpcConnection") + << info.str() << "\n" << DiagnosticInformation(ex); + + break; + } + } + } while (!m_ReaderHasError); +} + +void JsonRpcConnection::ShutdownStreamOnce(boost::asio::yield_context& yc) +{ + if (!--m_RunningCoroutines) { + try { + m_Stream->next_layer().async_shutdown(yc); + } catch (...) { + // https://stackoverflow.com/questions/130117/throwing-exceptions-out-of-a-destructor + } + + Log(LogWarning, "JsonRpcConnection") + << "API client disconnected for identity '" << m_Identity << "'"; + + if (m_Endpoint) { + m_Endpoint->RemoveClient(this); + } else { + auto listener (ApiListener::GetInstance()); + listener->RemoveAnonymousClient(this); + } + } } double JsonRpcConnection::GetTimestamp() const @@ -83,7 +160,7 @@ Endpoint::Ptr JsonRpcConnection::GetEndpoint() const return m_Endpoint; } -TlsStream::Ptr JsonRpcConnection::GetStream() const +std::shared_ptr JsonRpcConnection::GetStream() const { return m_Stream; } @@ -95,69 +172,16 @@ ConnectionRole JsonRpcConnection::GetRole() const void JsonRpcConnection::SendMessage(const Dictionary::Ptr& message) { - try { - ObjectLock olock(m_Stream); - - if (m_Stream->IsEof()) - return; - - size_t bytesSent = JsonRpc::SendMessage(m_Stream, message); - - if (m_Endpoint) - m_Endpoint->AddMessageSent(bytesSent); - - } catch (const std::exception& ex) { - std::ostringstream info; - info << "Error while sending JSON-RPC message for identity '" << m_Identity << "'"; - Log(LogWarning, "JsonRpcConnection") - << info.str() << "\n" << DiagnosticInformation(ex); - - Disconnect(); - } -} - -void JsonRpcConnection::Disconnect() -{ - Log(LogWarning, "JsonRpcConnection") - << "API client disconnected for identity '" << m_Identity << "'"; - - m_Stream->Close(); - - if (m_Endpoint) - m_Endpoint->RemoveClient(this); - else { - ApiListener::Ptr listener = ApiListener::GetInstance(); - listener->RemoveAnonymousClient(this); - } -} - -void JsonRpcConnection::MessageHandlerWrapper(const String& jsonString) -{ - if (m_Stream->IsEof()) - return; - - try { - MessageHandler(jsonString); - } catch (const std::exception& ex) { - Log(LogWarning, "JsonRpcConnection") - << "Error while reading JSON-RPC message for identity '" << m_Identity - << "': " << DiagnosticInformation(ex); - - Disconnect(); - - return; - } + m_IoStrand.post([this, message]() { + m_OutgoingMessagesQueue.emplace_back(message); + m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + }); } void JsonRpcConnection::MessageHandler(const String& jsonString) { Dictionary::Ptr message = JsonRpc::DecodeMessage(jsonString); - m_Seen = Utility::GetTime(); - - if (m_HeartbeatTimeout != 0) - m_NextHeartbeat = Utility::GetTime() + m_HeartbeatTimeout; - if (m_Endpoint && message->Contains("ts")) { double ts = message->Get("ts"); @@ -225,59 +249,12 @@ void JsonRpcConnection::MessageHandler(const String& jsonString) if (message->Contains("id")) { resultMessage->Set("jsonrpc", "2.0"); resultMessage->Set("id", message->Get("id")); - SendMessage(resultMessage); + + m_OutgoingMessagesQueue.emplace_back(resultMessage); + m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); } } -bool JsonRpcConnection::ProcessMessage() -{ - /* Limit for anonymous clients (signing requests and not configured endpoints. */ - ssize_t maxMessageLength = 1024 * 1024; - - if (m_Endpoint) - maxMessageLength = -1; /* no limit */ - - String message; - - StreamReadStatus srs = JsonRpc::ReadMessage(m_Stream, &message, m_Context, false, maxMessageLength); - - if (srs != StatusNewItem) - return false; - - l_JsonRpcConnectionWorkQueues[m_ID % l_JsonRpcConnectionWorkQueueCount].Enqueue(std::bind(&JsonRpcConnection::MessageHandlerWrapper, JsonRpcConnection::Ptr(this), message)); - - return true; -} - -void JsonRpcConnection::DataAvailableHandler() -{ - bool close = false; - - if (!m_Stream) - return; - - if (!m_Stream->IsEof()) { - boost::mutex::scoped_lock lock(m_DataHandlerMutex); - - try { - while (ProcessMessage()) - ; /* empty loop body */ - } catch (const std::exception& ex) { - Log(LogWarning, "JsonRpcConnection") - << "Error while reading JSON-RPC message for identity '" << m_Identity - << "': " << DiagnosticInformation(ex); - - Disconnect(); - - return; - } - } else - close = true; - - if (close) - Disconnect(); -} - Value SetLogPositionHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { double log_position = params->Get("log_position"); @@ -292,57 +269,3 @@ Value SetLogPositionHandler(const MessageOrigin::Ptr& origin, const Dictionary:: return Empty; } -void JsonRpcConnection::CheckLiveness() -{ - if (m_Seen < Utility::GetTime() - 60 && (!m_Endpoint || !m_Endpoint->GetSyncing())) { - Log(LogInformation, "JsonRpcConnection") - << "No messages for identity '" << m_Identity << "' have been received in the last 60 seconds."; - Disconnect(); - } -} - -void JsonRpcConnection::TimeoutTimerHandler() -{ - ApiListener::Ptr listener = ApiListener::GetInstance(); - - for (const JsonRpcConnection::Ptr& client : listener->GetAnonymousClients()) { - client->CheckLiveness(); - } - - for (const Endpoint::Ptr& endpoint : ConfigType::GetObjectsByType()) { - for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { - client->CheckLiveness(); - } - } -} - -size_t JsonRpcConnection::GetWorkQueueCount() -{ - return l_JsonRpcConnectionWorkQueueCount; -} - -size_t JsonRpcConnection::GetWorkQueueLength() -{ - size_t itemCount = 0; - - for (size_t i = 0; i < GetWorkQueueCount(); i++) - itemCount += l_JsonRpcConnectionWorkQueues[i].GetLength(); - - return itemCount; -} - -double JsonRpcConnection::GetWorkQueueRate() -{ - double rate = 0.0; - size_t count = GetWorkQueueCount(); - - /* If this is a standalone environment, we don't have any queues. */ - if (count == 0) - return 0.0; - - for (size_t i = 0; i < count; i++) - rate += l_JsonRpcConnectionWorkQueues[i].GetTaskCount(60) / 60.0; - - return rate / count; -} - diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 40df9af13..5263b30ac 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -8,6 +8,11 @@ #include "base/tlsstream.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" +#include +#include +#include +#include +#include namespace icinga { @@ -36,7 +41,7 @@ class JsonRpcConnection final : public Object public: DECLARE_PTR_TYPEDEFS(JsonRpcConnection); - JsonRpcConnection(const String& identity, bool authenticated, TlsStream::Ptr stream, ConnectionRole role); + JsonRpcConnection(const String& identity, bool authenticated, const std::shared_ptr& stream, ConnectionRole role); void Start(); @@ -44,45 +49,34 @@ public: String GetIdentity() const; bool IsAuthenticated() const; Endpoint::Ptr GetEndpoint() const; - TlsStream::Ptr GetStream() const; + std::shared_ptr GetStream() const; ConnectionRole GetRole() const; - void Disconnect(); - void SendMessage(const Dictionary::Ptr& request); - static void HeartbeatTimerHandler(); static Value HeartbeatAPIHandler(const intrusive_ptr& origin, const Dictionary::Ptr& params); - static size_t GetWorkQueueCount(); - static size_t GetWorkQueueLength(); - static double GetWorkQueueRate(); - static void SendCertificateRequest(const JsonRpcConnection::Ptr& aclient, const intrusive_ptr& origin, const String& path); private: - int m_ID; String m_Identity; bool m_Authenticated; Endpoint::Ptr m_Endpoint; - TlsStream::Ptr m_Stream; + std::shared_ptr m_Stream; ConnectionRole m_Role; double m_Timestamp; - double m_Seen; - double m_NextHeartbeat; - double m_HeartbeatTimeout; - boost::mutex m_DataHandlerMutex; + boost::asio::io_service::strand m_IoStrand; + std::vector m_OutgoingMessagesQueue; + boost::asio::deadline_timer m_OutgoingMessagesQueued; + bool m_ReaderHasError; + unsigned char m_RunningCoroutines; - StreamReadContext m_Context; + void HandleIncomingMessages(boost::asio::yield_context yc); + void WriteOutgoingMessages(boost::asio::yield_context yc); + void ShutdownStreamOnce(boost::asio::yield_context& yc); bool ProcessMessage(); - void MessageHandlerWrapper(const String& jsonString); void MessageHandler(const String& jsonString); - void DataAvailableHandler(); - - static void StaticInitialize(); - static void TimeoutTimerHandler(); - void CheckLiveness(); void CertificateRequestResponseHandler(const Dictionary::Ptr& message); }; From f9fff54da24815ae746b50c7b14ce6fe11fee6d0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 14:12:46 +0100 Subject: [PATCH 32/67] ApiListener: don't require a valid certificate for the TLS handshake to complete --- lib/remote/apilistener.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 87933ae7a..691009a9f 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -545,7 +545,7 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const verifyError = msgbuf.str(); } - return preverified; + return true; }); if (role == RoleClient) { From c46157d5524c924d19936cebf27ccd0e7eedd99e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 17:38:09 +0100 Subject: [PATCH 33/67] ApiListener: fix self-made security hole --- lib/remote/apilistener.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 691009a9f..235e6c573 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -531,13 +531,13 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const sslConn.set_verify_mode(ssl::verify_peer | ssl::verify_client_once); - bool verify_ok = false; + bool verify_ok = true; String verifyError; sslConn.set_verify_callback([&verify_ok, &verifyError](bool preverified, ssl::verify_context& ctx) { - verify_ok = preverified; - if (!preverified) { + verify_ok = false; + std::ostringstream msgbuf; int err = X509_STORE_CTX_get_error(ctx.native_handle()); From 2d16b02520ef60f523b249c576cb352fef3f1dc1 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 19 Feb 2019 18:06:14 +0100 Subject: [PATCH 34/67] ApiListener#NewClientHandlerInternal(): shut down TLS stream --- lib/remote/apilistener.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 235e6c573..56fda20e5 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -7,6 +7,7 @@ #include "remote/jsonrpc.hpp" #include "remote/apifunction.hpp" #include "base/convert.hpp" +#include "base/defer.hpp" #include "base/io-engine.hpp" #include "base/netstring.hpp" #include "base/json.hpp" @@ -570,6 +571,14 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const return; } + bool willBeShutDown = false; + + Defer shutDownIfNeeded ([&sslConn, &willBeShutDown, &yc]() { + if (!willBeShutDown) { + sslConn.async_shutdown(yc); + } + }); + std::shared_ptr cert (SSL_get_peer_certificate(sslConn.native_handle()), X509_free); String identity; Endpoint::Ptr endpoint; @@ -684,6 +693,8 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const if (aclient) { aclient->Start(); + + willBeShutDown = true; } } else { Log(LogNotice, "ApiListener", "New HTTP client"); @@ -691,6 +702,8 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, client); AddHttpClient(aclient); aclient->Start(); + + willBeShutDown = true; } } From 84b411501b9736d4b1b4489f8d43e79f128ff6dc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 12:00:11 +0100 Subject: [PATCH 35/67] Re-add JsonRpcConnection#Disconnect() --- lib/remote/apilistener.cpp | 12 ++++ lib/remote/jsonrpcconnection.cpp | 109 +++++++++++++++++++------------ lib/remote/jsonrpcconnection.hpp | 7 +- 3 files changed, 82 insertions(+), 46 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 56fda20e5..dccb8867f 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -213,6 +213,16 @@ void ApiListener::UpdateSSLContext() } m_SSLContext = context; + + for (const Endpoint::Ptr& endpoint : ConfigType::GetObjectsByType()) { + for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { + client->Disconnect(); + } + } + + for (const JsonRpcConnection::Ptr& client : m_AnonymousClients) { + client->Disconnect(); + } } void ApiListener::OnAllConfigLoaded() @@ -841,6 +851,8 @@ void ApiListener::ApiTimerHandler() for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) { if (client->GetTimestamp() == maxTs) { client->SendMessage(lmessage); + } else { + client->Disconnect(); } } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 3840a8c89..ff2e67ebf 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -27,32 +27,28 @@ JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, const std::shared_ptr& stream, ConnectionRole role) : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), m_Role(role), m_Timestamp(Utility::GetTime()), m_IoStrand(stream->get_io_service()), - m_OutgoingMessagesQueued(stream->get_io_service()), m_ReaderHasError(false), m_RunningCoroutines(0) + m_OutgoingMessagesQueued(stream->get_io_service()), m_WriterDone(stream->get_io_service()), m_ShuttingDown(false) { if (authenticated) m_Endpoint = Endpoint::GetByName(identity); m_OutgoingMessagesQueued.expires_at(boost::posix_time::pos_infin); + m_WriterDone.expires_at(boost::posix_time::pos_infin); } void JsonRpcConnection::Start() { namespace asio = boost::asio; - m_RunningCoroutines = 2; + JsonRpcConnection::Ptr preventGc (this); - asio::spawn(m_IoStrand, [this](asio::yield_context yc) { HandleIncomingMessages(yc); }); - asio::spawn(m_IoStrand, [this](asio::yield_context yc) { WriteOutgoingMessages(yc); }); + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleIncomingMessages(yc); }); + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { WriteOutgoingMessages(yc); }); } void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) { - Defer shutdownStreamOnce ([this, &yc]() { - m_ReaderHasError = true; - m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); - - ShutdownStreamOnce(yc); - }); + Defer disconnect ([this]() { Disconnect(); }); for (;;) { String message; @@ -60,9 +56,11 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) try { message = JsonRpc::ReadMessage(m_Stream, yc, m_Endpoint ? -1 : 1024 * 1024); } catch (const std::exception& ex) { - Log(LogWarning, "JsonRpcConnection") - << "Error while reading JSON-RPC message for identity '" << m_Identity - << "': " << DiagnosticInformation(ex); + if (!m_ShuttingDown) { + Log(LogWarning, "JsonRpcConnection") + << "Error while reading JSON-RPC message for identity '" << m_Identity + << "': " << DiagnosticInformation(ex); + } break; } @@ -72,9 +70,11 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) MessageHandler(message); } catch (const std::exception& ex) { - Log(LogWarning, "JsonRpcConnection") - << "Error while processing JSON-RPC message for identity '" << m_Identity - << "': " << DiagnosticInformation(ex); + if (!m_ShuttingDown) { + Log(LogWarning, "JsonRpcConnection") + << "Error while processing JSON-RPC message for identity '" << m_Identity + << "': " << DiagnosticInformation(ex); + } break; } @@ -83,7 +83,9 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) void JsonRpcConnection::WriteOutgoingMessages(boost::asio::yield_context yc) { - Defer shutdownStreamOnce ([this, &yc]() { ShutdownStreamOnce(yc); }); + Defer disconnect ([this]() { Disconnect(); }); + + Defer signalWriterDone ([this]() { m_WriterDone.expires_at(boost::posix_time::neg_infin); }); do { try { @@ -108,36 +110,17 @@ void JsonRpcConnection::WriteOutgoingMessages(boost::asio::yield_context yc) m_Stream->async_flush(yc); } catch (const std::exception& ex) { - std::ostringstream info; - info << "Error while sending JSON-RPC message for identity '" << m_Identity << "'"; - Log(LogWarning, "JsonRpcConnection") - << info.str() << "\n" << DiagnosticInformation(ex); + if (!m_ShuttingDown) { + std::ostringstream info; + info << "Error while sending JSON-RPC message for identity '" << m_Identity << "'"; + Log(LogWarning, "JsonRpcConnection") + << info.str() << "\n" << DiagnosticInformation(ex); + } break; } } - } while (!m_ReaderHasError); -} - -void JsonRpcConnection::ShutdownStreamOnce(boost::asio::yield_context& yc) -{ - if (!--m_RunningCoroutines) { - try { - m_Stream->next_layer().async_shutdown(yc); - } catch (...) { - // https://stackoverflow.com/questions/130117/throwing-exceptions-out-of-a-destructor - } - - Log(LogWarning, "JsonRpcConnection") - << "API client disconnected for identity '" << m_Identity << "'"; - - if (m_Endpoint) { - m_Endpoint->RemoveClient(this); - } else { - auto listener (ApiListener::GetInstance()); - listener->RemoveAnonymousClient(this); - } - } + } while (!m_ShuttingDown); } double JsonRpcConnection::GetTimestamp() const @@ -178,6 +161,46 @@ void JsonRpcConnection::SendMessage(const Dictionary::Ptr& message) }); } +void JsonRpcConnection::Disconnect() +{ + namespace asio = boost::asio; + + JsonRpcConnection::Ptr preventGc (this); + + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { + if (!m_ShuttingDown) { + m_ShuttingDown = true; + + Log(LogWarning, "JsonRpcConnection") + << "API client disconnected for identity '" << m_Identity << "'"; + + m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + + try { + m_WriterDone.async_wait(yc); + } catch (...) { + } + + try { + m_Stream->next_layer().async_shutdown(yc); + } catch (...) { + } + + try { + m_Stream->lowest_layer().shutdown(m_Stream->lowest_layer().shutdown_both); + } catch (...) { + } + + if (m_Endpoint) { + m_Endpoint->RemoveClient(this); + } else { + auto listener (ApiListener::GetInstance()); + listener->RemoveAnonymousClient(this); + } + } + }); +} + void JsonRpcConnection::MessageHandler(const String& jsonString) { Dictionary::Ptr message = JsonRpc::DecodeMessage(jsonString); diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 5263b30ac..51787244e 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -52,6 +52,8 @@ public: std::shared_ptr GetStream() const; ConnectionRole GetRole() const; + void Disconnect(); + void SendMessage(const Dictionary::Ptr& request); static Value HeartbeatAPIHandler(const intrusive_ptr& origin, const Dictionary::Ptr& params); @@ -68,12 +70,11 @@ private: boost::asio::io_service::strand m_IoStrand; std::vector m_OutgoingMessagesQueue; boost::asio::deadline_timer m_OutgoingMessagesQueued; - bool m_ReaderHasError; - unsigned char m_RunningCoroutines; + boost::asio::deadline_timer m_WriterDone; + bool m_ShuttingDown; void HandleIncomingMessages(boost::asio::yield_context yc); void WriteOutgoingMessages(boost::asio::yield_context yc); - void ShutdownStreamOnce(boost::asio::yield_context& yc); bool ProcessMessage(); void MessageHandler(const String& jsonString); From 7aae8bd2658e7b470740e24f50bfbaa980664a68 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 12:28:49 +0100 Subject: [PATCH 36/67] JsonRpcConnection: re-add heartbeats --- lib/remote/jsonrpcconnection-heartbeat.cpp | 42 ++++++++++++++++++++++ lib/remote/jsonrpcconnection.cpp | 3 +- lib/remote/jsonrpcconnection.hpp | 2 ++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 91fa51891..569ffbb1c 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -7,13 +7,55 @@ #include "base/configtype.hpp" #include "base/logger.hpp" #include "base/utility.hpp" +#include +#include +#include using namespace icinga; REGISTER_APIFUNCTION(Heartbeat, event, &JsonRpcConnection::HeartbeatAPIHandler); +void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) +{ + boost::asio::deadline_timer timer (m_Stream->get_io_service()); + + for (;;) { + timer.expires_from_now(boost::posix_time::seconds(10)); + timer.async_wait(yc); + + if (m_ShuttingDown) { + break; + } + + if (m_NextHeartbeat != 0 && m_NextHeartbeat < Utility::GetTime()) { + Log(LogWarning, "JsonRpcConnection") + << "Client for endpoint '" << m_Endpoint->GetName() << "' has requested " + << "heartbeat message but hasn't responded in time. Closing connection."; + + Disconnect(); + break; + } + + m_OutgoingMessagesQueue.emplace_back(new Dictionary({ + { "jsonrpc", "2.0" }, + { "method", "event::Heartbeat" }, + { "params", new Dictionary({ + { "timeout", 120 } + }) } + })); + + m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + } +} + Value JsonRpcConnection::HeartbeatAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { + Value vtimeout = params->Get("timeout"); + + if (!vtimeout.IsEmpty()) { + origin->FromClient->m_NextHeartbeat = Utility::GetTime() + vtimeout; + } + return Empty; } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index ff2e67ebf..4a50ea20c 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -26,7 +26,7 @@ REGISTER_APIFUNCTION(SetLogPosition, log, &SetLogPositionHandler); JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, const std::shared_ptr& stream, ConnectionRole role) : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), - m_Role(role), m_Timestamp(Utility::GetTime()), m_IoStrand(stream->get_io_service()), + m_Role(role), m_Timestamp(Utility::GetTime()), m_NextHeartbeat(0), m_IoStrand(stream->get_io_service()), m_OutgoingMessagesQueued(stream->get_io_service()), m_WriterDone(stream->get_io_service()), m_ShuttingDown(false) { if (authenticated) @@ -44,6 +44,7 @@ void JsonRpcConnection::Start() asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleIncomingMessages(yc); }); asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { WriteOutgoingMessages(yc); }); + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleAndWriteHeartbeats(yc); }); } void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 51787244e..54a71b889 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -67,6 +67,7 @@ private: std::shared_ptr m_Stream; ConnectionRole m_Role; double m_Timestamp; + double m_NextHeartbeat; boost::asio::io_service::strand m_IoStrand; std::vector m_OutgoingMessagesQueue; boost::asio::deadline_timer m_OutgoingMessagesQueued; @@ -75,6 +76,7 @@ private: void HandleIncomingMessages(boost::asio::yield_context yc); void WriteOutgoingMessages(boost::asio::yield_context yc); + void HandleAndWriteHeartbeats(boost::asio::yield_context yc); bool ProcessMessage(); void MessageHandler(const String& jsonString); From a54bd9d5c4c454cbd557199f01e4c88fab4c4edd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 13:49:50 +0100 Subject: [PATCH 37/67] JsonRpcConnection: re-add automatic disconnect --- lib/remote/jsonrpcconnection.cpp | 28 +++++++++++++++++++++++++++- lib/remote/jsonrpcconnection.hpp | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 4a50ea20c..73fcc6398 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -14,7 +14,9 @@ #include "base/tlsstream.hpp" #include #include +#include #include +#include #include #include @@ -26,7 +28,7 @@ REGISTER_APIFUNCTION(SetLogPosition, log, &SetLogPositionHandler); JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, const std::shared_ptr& stream, ConnectionRole role) : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), - m_Role(role), m_Timestamp(Utility::GetTime()), m_NextHeartbeat(0), m_IoStrand(stream->get_io_service()), + m_Role(role), m_Timestamp(Utility::GetTime()), m_Seen(Utility::GetTime()), m_NextHeartbeat(0), m_IoStrand(stream->get_io_service()), m_OutgoingMessagesQueued(stream->get_io_service()), m_WriterDone(stream->get_io_service()), m_ShuttingDown(false) { if (authenticated) @@ -45,6 +47,7 @@ void JsonRpcConnection::Start() asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleIncomingMessages(yc); }); asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { WriteOutgoingMessages(yc); }); asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleAndWriteHeartbeats(yc); }); + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { CheckLiveness(yc); }); } void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) @@ -66,6 +69,8 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) break; } + m_Seen = Utility::GetTime(); + try { CpuBoundWork handleMessage (yc); @@ -293,3 +298,24 @@ Value SetLogPositionHandler(const MessageOrigin::Ptr& origin, const Dictionary:: return Empty; } +void JsonRpcConnection::CheckLiveness(boost::asio::yield_context yc) +{ + boost::asio::deadline_timer timer (m_Stream->get_io_service()); + + for (;;) { + timer.expires_from_now(boost::posix_time::seconds(30)); + timer.async_wait(yc); + + if (m_ShuttingDown) { + break; + } + + if (m_Seen < Utility::GetTime() - 60 && (!m_Endpoint || !m_Endpoint->GetSyncing())) { + Log(LogInformation, "JsonRpcConnection") + << "No messages for identity '" << m_Identity << "' have been received in the last 60 seconds."; + + Disconnect(); + break; + } + } +} diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 54a71b889..13ee5f62d 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -67,6 +67,7 @@ private: std::shared_ptr m_Stream; ConnectionRole m_Role; double m_Timestamp; + double m_Seen; double m_NextHeartbeat; boost::asio::io_service::strand m_IoStrand; std::vector m_OutgoingMessagesQueue; @@ -77,6 +78,7 @@ private: void HandleIncomingMessages(boost::asio::yield_context yc); void WriteOutgoingMessages(boost::asio::yield_context yc); void HandleAndWriteHeartbeats(boost::asio::yield_context yc); + void CheckLiveness(boost::asio::yield_context yc); bool ProcessMessage(); void MessageHandler(const String& jsonString); From a451327b816684dd5a525f22b09f668f7c6a404e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 14:24:09 +0100 Subject: [PATCH 38/67] JsonRpcConnection: re-add num_json_rpc_work_queue_item_rate --- lib/remote/apilistener.cpp | 3 +++ lib/remote/jsonrpcconnection.cpp | 9 +++++++++ lib/remote/jsonrpcconnection.hpp | 2 ++ 3 files changed, 14 insertions(+) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index dccb8867f..ebe18358b 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -1466,6 +1466,7 @@ std::pair ApiListener::GetStatus() size_t httpClients = GetHttpClients().size(); size_t syncQueueItems = m_SyncQueue.GetLength(); size_t relayQueueItems = m_RelayQueue.GetLength(); + double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate(); double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0; double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0; @@ -1483,6 +1484,7 @@ std::pair ApiListener::GetStatus() { "anonymous_clients", jsonRpcAnonymousClients }, { "sync_queue_items", syncQueueItems }, { "relay_queue_items", relayQueueItems }, + { "work_queue_item_rate", workQueueItemRate }, { "sync_queue_item_rate", syncQueueItemRate }, { "relay_queue_item_rate", relayQueueItemRate } }) }, @@ -1502,6 +1504,7 @@ std::pair ApiListener::GetStatus() perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems); perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems); + perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate); perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate); perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate); diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 73fcc6398..d529427f0 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -25,6 +25,8 @@ using namespace icinga; static Value SetLogPositionHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); REGISTER_APIFUNCTION(SetLogPosition, log, &SetLogPositionHandler); +static RingBuffer l_TaskStats (15 * 60); + JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, const std::shared_ptr& stream, ConnectionRole role) : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), @@ -84,6 +86,8 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) break; } + + l_TaskStats.InsertValue(Utility::GetTime(), 1); } } @@ -319,3 +323,8 @@ void JsonRpcConnection::CheckLiveness(boost::asio::yield_context yc) } } } + +double JsonRpcConnection::GetWorkQueueRate() +{ + return l_TaskStats.UpdateAndGetValues(Utility::GetTime(), 60) / 60.0; +} diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 13ee5f62d..bc5fa398d 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -58,6 +58,8 @@ public: static Value HeartbeatAPIHandler(const intrusive_ptr& origin, const Dictionary::Ptr& params); + static double GetWorkQueueRate(); + static void SendCertificateRequest(const JsonRpcConnection::Ptr& aclient, const intrusive_ptr& origin, const String& path); private: From 16913cb9771c4d1f362b15277a9e2bc747ab408e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 14:41:01 +0100 Subject: [PATCH 39/67] JsonRpcConnection: add missing CpuBoundWork --- lib/remote/jsonrpcconnection.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index d529427f0..fa8b65ffc 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -87,6 +87,8 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) break; } + CpuBoundWork taskStats (yc); + l_TaskStats.InsertValue(Utility::GetTime(), 1); } } @@ -201,6 +203,8 @@ void JsonRpcConnection::Disconnect() } catch (...) { } + CpuBoundWork removeClient (yc); + if (m_Endpoint) { m_Endpoint->RemoveClient(this); } else { From f029fd48847c797cee32d8068745b4a1e6fe0f08 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 14:56:12 +0100 Subject: [PATCH 40/67] Re-add HttpServerConnection#Disconnect() --- lib/remote/httpserverconnection.cpp | 65 +++++++++++++++++++---------- lib/remote/httpserverconnection.hpp | 4 ++ 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index a37b3a0b1..2419eb595 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -12,7 +12,6 @@ #include "base/configtype.hpp" #include "base/defer.hpp" #include "base/exception.hpp" -#include "base/io-engine.hpp" #include "base/logger.hpp" #include "base/objectlock.hpp" #include "base/timer.hpp" @@ -31,7 +30,7 @@ using namespace icinga; auto const l_ServerHeader ("Icinga/" + Application::GetAppVersion()); HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const std::shared_ptr& stream) - : m_Stream(stream) + : m_Stream(stream), m_IoStrand(stream->get_io_service()), m_ShuttingDown(false) { if (authenticated) { m_ApiUser = ApiUser::GetByClientCN(identity); @@ -51,7 +50,43 @@ void HttpServerConnection::Start() { namespace asio = boost::asio; - asio::spawn(IoEngine::Get().GetIoService(), [this](asio::yield_context yc) { ProcessMessages(yc); }); + HttpServerConnection::Ptr preventGc (this); + + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { ProcessMessages(yc); }); +} + +void HttpServerConnection::Disconnect() +{ + namespace asio = boost::asio; + + HttpServerConnection::Ptr preventGc (this); + + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { + if (!m_ShuttingDown) { + m_ShuttingDown = true; + + Log(LogInformation, "HttpServerConnection") + << "HTTP client disconnected (from " << m_PeerAddress << ")"; + + try { + m_Stream->next_layer().async_shutdown(yc); + } catch (...) { + } + + try { + m_Stream->lowest_layer().shutdown(m_Stream->lowest_layer().shutdown_both); + } catch (...) { + } + + auto listener (ApiListener::GetInstance()); + + if (listener) { + CpuBoundWork removeHttpClient (yc); + + listener->RemoveHttpClient(this); + } + } + }); } static inline @@ -357,23 +392,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) namespace beast = boost::beast; namespace http = beast::http; - Defer removeHttpClient ([this, &yc]() { - auto listener (ApiListener::GetInstance()); - - if (listener) { - CpuBoundWork removeHttpClient (yc); - - listener->RemoveHttpClient(this); - } - }); - - Defer shutdown ([this, &yc]() { - try { - m_Stream->next_layer().async_shutdown(yc); - } catch (...) { - // https://stackoverflow.com/questions/130117/throwing-exceptions-out-of-a-destructor - } - }); + Defer disconnect ([this]() { Disconnect(); }); try { beast::flat_buffer buf; @@ -440,7 +459,9 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) } } } catch (const std::exception& ex) { - Log(LogCritical, "HttpServerConnection") - << "Unhandled exception while processing HTTP request: " << DiagnosticInformation(ex); + if (!m_ShuttingDown) { + Log(LogCritical, "HttpServerConnection") + << "Unhandled exception while processing HTTP request: " << DiagnosticInformation(ex); + } } } diff --git a/lib/remote/httpserverconnection.hpp b/lib/remote/httpserverconnection.hpp index 3fdbeef50..bb3291991 100644 --- a/lib/remote/httpserverconnection.hpp +++ b/lib/remote/httpserverconnection.hpp @@ -7,6 +7,7 @@ #include "base/string.hpp" #include "base/tlsstream.hpp" #include +#include #include namespace icinga @@ -25,11 +26,14 @@ public: HttpServerConnection(const String& identity, bool authenticated, const std::shared_ptr& stream); void Start(); + void Disconnect(); private: ApiUser::Ptr m_ApiUser; std::shared_ptr m_Stream; String m_PeerAddress; + boost::asio::io_service::strand m_IoStrand; + bool m_ShuttingDown; void ProcessMessages(boost::asio::yield_context yc); }; From 87b0c452db83eb2413415b6cb4d90ee251cc114c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 15:27:15 +0100 Subject: [PATCH 41/67] HttpServerConnection: re-add automatic disconnect --- lib/remote/httpserverconnection.cpp | 35 +++++++++++++++++++++++++++-- lib/remote/httpserverconnection.hpp | 2 ++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 2419eb595..198a472b9 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -17,6 +17,7 @@ #include "base/timer.hpp" #include "base/tlsstream.hpp" #include "base/utility.hpp" +#include #include #include #include @@ -30,7 +31,7 @@ using namespace icinga; auto const l_ServerHeader ("Icinga/" + Application::GetAppVersion()); HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const std::shared_ptr& stream) - : m_Stream(stream), m_IoStrand(stream->get_io_service()), m_ShuttingDown(false) + : m_Stream(stream), m_Seen(Utility::GetTime()), m_IoStrand(stream->get_io_service()), m_ShuttingDown(false) { if (authenticated) { m_ApiUser = ApiUser::GetByClientCN(identity); @@ -53,6 +54,7 @@ void HttpServerConnection::Start() HttpServerConnection::Ptr preventGc (this); asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { ProcessMessages(yc); }); + asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { CheckLiveness(yc); }); } void HttpServerConnection::Disconnect() @@ -351,6 +353,7 @@ bool ProcessRequest( boost::beast::http::request& request, ApiUser::Ptr& authenticatedUser, boost::beast::http::response& response, + double& seen, boost::asio::yield_context& yc ) { @@ -361,6 +364,8 @@ bool ProcessRequest( try { CpuBoundWork handlingRequest (yc); + Defer updateSeen ([&seen]() { seen = Utility::GetTime(); }); + HttpHandler::ProcessRequest(stream, authenticatedUser, request, response, yc, hasStartedStreaming); } catch (const std::exception& ex) { if (hasStartedStreaming) { @@ -409,6 +414,8 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) break; } + m_Seen = Utility::GetTime(); + auto& request (parser.get()); { @@ -450,7 +457,9 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) break; } - if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, yc)) { + m_Seen = std::numeric_limits::max(); + + if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, m_Seen, yc)) { break; } @@ -465,3 +474,25 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) } } } + +void HttpServerConnection::CheckLiveness(boost::asio::yield_context yc) +{ + boost::asio::deadline_timer timer (m_Stream->get_io_service()); + + for (;;) { + timer.expires_from_now(boost::posix_time::seconds(5)); + timer.async_wait(yc); + + if (m_ShuttingDown) { + break; + } + + if (m_Seen < Utility::GetTime() - 10) { + Log(LogInformation, "HttpServerConnection") + << "No messages for HTTP connection have been received in the last 10 seconds."; + + Disconnect(); + break; + } + } +} diff --git a/lib/remote/httpserverconnection.hpp b/lib/remote/httpserverconnection.hpp index bb3291991..7db97ed68 100644 --- a/lib/remote/httpserverconnection.hpp +++ b/lib/remote/httpserverconnection.hpp @@ -31,11 +31,13 @@ public: private: ApiUser::Ptr m_ApiUser; std::shared_ptr m_Stream; + double m_Seen; String m_PeerAddress; boost::asio::io_service::strand m_IoStrand; bool m_ShuttingDown; void ProcessMessages(boost::asio::yield_context yc); + void CheckLiveness(boost::asio::yield_context yc); }; } From 19625e62efa3895273a651327356f56ecf1cee9f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 20 Feb 2019 16:07:41 +0100 Subject: [PATCH 42/67] ApiListener: fix self-made security hole --- lib/remote/apilistener.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index ebe18358b..d56c532bf 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -629,6 +629,8 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const log << " (no Endpoint object found for identity)"; } } else { + verify_ok = false; + Log(LogInformation, "ApiListener") << "New client connection " << conninfo << " (no client certificate)"; } From e26774c7f803c706d904618b67d22de9adb2c0c0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 21 Feb 2019 10:08:38 +0100 Subject: [PATCH 43/67] IoEngine: adjust I/O threads --- lib/base/io-engine.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index 0079ca251..c93a4d87a 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -99,17 +99,10 @@ boost::asio::io_service& IoEngine::GetIoService() return m_IoService; } -IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_Threads(decltype(m_Threads)::size_type(std::thread::hardware_concurrency())), m_AlreadyExpiredTimer(m_IoService) +IoEngine::IoEngine() : m_IoService(), m_KeepAlive(m_IoService), m_Threads(decltype(m_Threads)::size_type(std::thread::hardware_concurrency() * 2u)), m_AlreadyExpiredTimer(m_IoService) { m_AlreadyExpiredTimer.expires_at(boost::posix_time::neg_infin); - - auto concurrency (std::thread::hardware_concurrency()); - - if (concurrency < 2) { - m_CpuBoundSemaphore.store(1); - } else { - m_CpuBoundSemaphore.store(concurrency - 1u); - } + m_CpuBoundSemaphore.store(std::thread::hardware_concurrency() * 3u / 2u); for (auto& thread : m_Threads) { thread = std::thread(&IoEngine::RunEventLoop, this); From b5fddaf3ce74d865383462aed035854f8271bbcb Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 21 Feb 2019 10:41:31 +0100 Subject: [PATCH 44/67] ApiListener: log why bind(2) failed --- lib/remote/apilistener.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index d56c532bf..4928c285c 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -385,9 +385,9 @@ bool ApiListener::AddListener(const String& node, const String& service) } } } - } catch (const std::exception&) { + } catch (const std::exception& ex) { Log(LogCritical, "ApiListener") - << "Cannot bind TCP socket for host '" << node << "' on port '" << service << "'."; + << "Cannot bind TCP socket for host '" << node << "' on port '" << service << "': " << DiagnosticInformation(ex, false); return false; } From 326bf6625578180dc5a21ebcceb2043fd538714b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 21 Feb 2019 10:55:47 +0100 Subject: [PATCH 45/67] ApiListener: use setsockopt(), not tcp::acceptor#set_option() --- lib/remote/apilistener.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 4928c285c..f3057c289 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -22,7 +22,6 @@ #include "base/exception.hpp" #include #include -#include #include #include #include @@ -370,8 +369,20 @@ bool ApiListener::AddListener(const String& node, const String& service) for (;;) { try { acceptor->open(current->endpoint().protocol()); - acceptor->set_option(ip::v6_only(false)); - acceptor->set_option(tcp::acceptor::reuse_address(true)); + + { + auto fd (acceptor->native_handle()); + + const int optFalse = 0; + setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&optFalse), sizeof(optFalse)); + + const int optTrue = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optTrue), sizeof(optTrue)); +#ifndef _WIN32 + setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&optTrue), sizeof(optTrue)); +#endif /* _WIN32 */ + } + acceptor->bind(current->endpoint()); break; From b384f859c9002e9b190193b658e1a99276d87191 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Feb 2019 15:07:59 +0100 Subject: [PATCH 46/67] Make IoEngine::m_CpuBoundSemaphore signed --- lib/base/io-engine.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index 05610ca6f..65059db68 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -102,7 +102,7 @@ private: boost::asio::io_service::work m_KeepAlive; std::vector m_Threads; boost::asio::deadline_timer m_AlreadyExpiredTimer; - std::atomic_uint_fast32_t m_CpuBoundSemaphore; + std::atomic_int_fast32_t m_CpuBoundSemaphore; }; class TerminateIoThread : public std::exception From e129c561d58386a9992507372dbce73ee64c7cbe Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Feb 2019 15:38:02 +0100 Subject: [PATCH 47/67] HttpServerConnection: don't disconnect during sending response --- lib/remote/httpserverconnection.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 198a472b9..af2a69b42 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -353,7 +353,6 @@ bool ProcessRequest( boost::beast::http::request& request, ApiUser::Ptr& authenticatedUser, boost::beast::http::response& response, - double& seen, boost::asio::yield_context& yc ) { @@ -364,8 +363,6 @@ bool ProcessRequest( try { CpuBoundWork handlingRequest (yc); - Defer updateSeen ([&seen]() { seen = Utility::GetTime(); }); - HttpHandler::ProcessRequest(stream, authenticatedUser, request, response, yc, hasStartedStreaming); } catch (const std::exception& ex) { if (hasStartedStreaming) { @@ -403,6 +400,8 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) beast::flat_buffer buf; for (;;) { + m_Seen = Utility::GetTime(); + http::parser parser; http::response response; @@ -459,7 +458,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) m_Seen = std::numeric_limits::max(); - if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, m_Seen, yc)) { + if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, yc)) { break; } From d3392d157922c5ed881a8c9e0a6fdc1d62ecd13d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Feb 2019 15:42:48 +0100 Subject: [PATCH 48/67] Rename AsioTlsStreamHack to UnbufferedAsioTlsStream --- lib/base/tlsstream.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index cca8cb286..0dfb4e312 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -99,17 +99,17 @@ private: void CloseInternal(bool inDestructor); }; -class AsioTlsStreamHack : public boost::asio::ssl::stream +class UnbufferedAsioTlsStream : public boost::asio::ssl::stream { public: inline - AsioTlsStreamHack(std::pair& init) + UnbufferedAsioTlsStream(std::pair& init) : stream(*init.first, *init.second) { } }; -class AsioTlsStream : public boost::asio::buffered_stream +class AsioTlsStream : public boost::asio::buffered_stream { public: inline From 8b3efe5759c855b973178b181f96a9bdfbaf0bcf Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Feb 2019 16:13:28 +0100 Subject: [PATCH 49/67] Introduce AsioConditionVariable --- lib/base/io-engine.cpp | 23 +++++++++++++++++++++ lib/base/io-engine.hpp | 18 ++++++++++++++++ lib/remote/jsonrpcconnection-heartbeat.cpp | 2 +- lib/remote/jsonrpcconnection.cpp | 24 +++++++--------------- lib/remote/jsonrpcconnection.hpp | 6 +++--- 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index c93a4d87a..b70050552 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -27,6 +27,7 @@ #include #include #include +#include using namespace icinga; @@ -137,3 +138,25 @@ void IoEngine::RunEventLoop() } } } + +AsioConditionVariable::AsioConditionVariable(boost::asio::io_service& io, bool init) + : m_Timer(io) +{ + m_Timer.expires_at(init ? boost::posix_time::neg_infin : boost::posix_time::pos_infin); +} + +void AsioConditionVariable::Set() +{ + m_Timer.expires_at(boost::posix_time::neg_infin); +} + +void AsioConditionVariable::Clear() +{ + m_Timer.expires_at(boost::posix_time::pos_infin); +} + +void AsioConditionVariable::Wait(boost::asio::yield_context yc) +{ + boost::system::error_code ec; + m_Timer.async_wait(yc[ec]); +} diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index 65059db68..d08525081 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -109,4 +109,22 @@ class TerminateIoThread : public std::exception { }; +/** + * Condition variable which doesn't block I/O threads + * + * @ingroup base + */ +class AsioConditionVariable +{ +public: + AsioConditionVariable(boost::asio::io_service& io, bool init = false); + + void Set(); + void Clear(); + void Wait(boost::asio::yield_context yc); + +private: + boost::asio::deadline_timer m_Timer; +}; + #endif /* IO_ENGINE_H */ diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 569ffbb1c..da6afe4b7 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -44,7 +44,7 @@ void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) }) } })); - m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + m_OutgoingMessagesQueued.Set(); } } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index fa8b65ffc..a0f37f950 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include using namespace icinga; @@ -35,9 +34,6 @@ JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, { if (authenticated) m_Endpoint = Endpoint::GetByName(identity); - - m_OutgoingMessagesQueued.expires_at(boost::posix_time::pos_infin); - m_WriterDone.expires_at(boost::posix_time::pos_infin); } void JsonRpcConnection::Start() @@ -97,18 +93,15 @@ void JsonRpcConnection::WriteOutgoingMessages(boost::asio::yield_context yc) { Defer disconnect ([this]() { Disconnect(); }); - Defer signalWriterDone ([this]() { m_WriterDone.expires_at(boost::posix_time::neg_infin); }); + Defer signalWriterDone ([this]() { m_WriterDone.Set(); }); do { - try { - m_OutgoingMessagesQueued.async_wait(yc); - } catch (...) { - } + m_OutgoingMessagesQueued.Wait(yc); auto queue (std::move(m_OutgoingMessagesQueue)); m_OutgoingMessagesQueue.clear(); - m_OutgoingMessagesQueued.expires_at(boost::posix_time::pos_infin); + m_OutgoingMessagesQueued.Clear(); if (!queue.empty()) { try { @@ -169,7 +162,7 @@ void JsonRpcConnection::SendMessage(const Dictionary::Ptr& message) { m_IoStrand.post([this, message]() { m_OutgoingMessagesQueue.emplace_back(message); - m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + m_OutgoingMessagesQueued.Set(); }); } @@ -186,12 +179,9 @@ void JsonRpcConnection::Disconnect() Log(LogWarning, "JsonRpcConnection") << "API client disconnected for identity '" << m_Identity << "'"; - m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + m_OutgoingMessagesQueued.Set(); - try { - m_WriterDone.async_wait(yc); - } catch (...) { - } + m_WriterDone.Wait(yc); try { m_Stream->next_layer().async_shutdown(yc); @@ -288,7 +278,7 @@ void JsonRpcConnection::MessageHandler(const String& jsonString) resultMessage->Set("id", message->Get("id")); m_OutgoingMessagesQueue.emplace_back(resultMessage); - m_OutgoingMessagesQueued.expires_at(boost::posix_time::neg_infin); + m_OutgoingMessagesQueued.Set(); } } diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index bc5fa398d..b0679d368 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -5,6 +5,7 @@ #include "remote/i2-remote.hpp" #include "remote/endpoint.hpp" +#include "base/io-engine.hpp" #include "base/tlsstream.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" @@ -12,7 +13,6 @@ #include #include #include -#include namespace icinga { @@ -73,8 +73,8 @@ private: double m_NextHeartbeat; boost::asio::io_service::strand m_IoStrand; std::vector m_OutgoingMessagesQueue; - boost::asio::deadline_timer m_OutgoingMessagesQueued; - boost::asio::deadline_timer m_WriterDone; + AsioConditionVariable m_OutgoingMessagesQueued; + AsioConditionVariable m_WriterDone; bool m_ShuttingDown; void HandleIncomingMessages(boost::asio::yield_context yc); From 79220ee6470577ede17532a4cc38d39a7b731086 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Feb 2019 16:16:59 +0100 Subject: [PATCH 50/67] io-engine.hpp: fix missing namespace --- lib/base/io-engine.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index d08525081..ebb498818 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -30,6 +30,9 @@ #include #include +namespace icinga +{ + /** * Scope lock for CPU-bound work done in an I/O thread * @@ -127,4 +130,6 @@ private: boost::asio::deadline_timer m_Timer; }; +} + #endif /* IO_ENGINE_H */ From e6d78bf3610f473ba99c7445e65af194440c645e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 25 Feb 2019 16:18:48 +0100 Subject: [PATCH 51/67] Move some TCP/TLS logic out of ApiListener ... for re-using it --- lib/base/tcpsocket.hpp | 31 ++++++++++++++++ lib/base/tlsstream.cpp | 49 ++++++++++++++++++++++++++ lib/base/tlsstream.hpp | 48 ++++++++++++++++++++++--- lib/remote/apilistener.cpp | 72 +++++--------------------------------- 4 files changed, 131 insertions(+), 69 deletions(-) diff --git a/lib/base/tcpsocket.hpp b/lib/base/tcpsocket.hpp index 4f8a2a0f0..069288aad 100644 --- a/lib/base/tcpsocket.hpp +++ b/lib/base/tcpsocket.hpp @@ -5,6 +5,8 @@ #include "base/i2-base.hpp" #include "base/socket.hpp" +#include +#include namespace icinga { @@ -25,6 +27,35 @@ public: void Connect(const String& node, const String& service); }; +template +void Connect(Socket& socket, const String& node, const String& service, boost::asio::yield_context yc) +{ + using boost::asio::ip::tcp; + + tcp::resolver resolver (socket.get_io_service()); + tcp::resolver::query query (node, service); + auto result (resolver.async_resolve(query, yc)); + auto current (result.begin()); + + for (;;) { + try { + socket.open(current->endpoint().protocol()); + socket.set_option(tcp::socket::keep_alive(true)); + socket.async_connect(current->endpoint(), yc); + + break; + } catch (const std::exception&) { + if (++current == result.end()) { + throw; + } + + if (socket.is_open()) { + socket.close(); + } + } + } +} + } #endif /* TCPSOCKET_H */ diff --git a/lib/base/tlsstream.cpp b/lib/base/tlsstream.cpp index 129d0bc74..1693b9edc 100644 --- a/lib/base/tlsstream.cpp +++ b/lib/base/tlsstream.cpp @@ -7,7 +7,13 @@ #include "base/configuration.hpp" #include "base/convert.hpp" #include +#include +#include #include +#include +#include +#include +#include #ifndef _WIN32 # include @@ -447,3 +453,46 @@ Socket::Ptr TlsStream::GetSocket() const { return m_Socket; } + +bool UnbufferedAsioTlsStream::IsVerifyOK() const +{ + return m_VerifyOK; +} + +String UnbufferedAsioTlsStream::GetVerifyError() const +{ + return m_VerifyError; +} + +void UnbufferedAsioTlsStream::BeforeHandshake(handshake_type type) +{ + namespace ssl = boost::asio::ssl; + + set_verify_mode(ssl::verify_peer | ssl::verify_client_once); + + set_verify_callback([this](bool preverified, ssl::verify_context& ctx) { + if (!preverified) { + m_VerifyOK = false; + + std::ostringstream msgbuf; + int err = X509_STORE_CTX_get_error(ctx.native_handle()); + + msgbuf << "code " << err << ": " << X509_verify_cert_error_string(err); + m_VerifyError = msgbuf.str(); + } + + return true; + }); + +#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME + if (type == client && !m_Hostname.IsEmpty()) { + String environmentName = Application::GetAppEnvironment(); + String serverName = m_Hostname; + + if (!environmentName.IsEmpty()) + serverName += ":" + environmentName; + + SSL_set_tlsext_host_name(native_handle(), serverName.CStr()); + } +#endif /* SSL_CTRL_SET_TLSEXT_HOSTNAME */ +} diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index 0dfb4e312..2432d16e8 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -99,28 +99,66 @@ private: void CloseInternal(bool inDestructor); }; +struct UnbufferedAsioTlsStreamParams +{ + boost::asio::io_service& IoService; + boost::asio::ssl::context& SslContext; + const String& Hostname; +}; + class UnbufferedAsioTlsStream : public boost::asio::ssl::stream { +private: + typedef boost::asio::ssl::stream Parent; + public: inline - UnbufferedAsioTlsStream(std::pair& init) - : stream(*init.first, *init.second) + UnbufferedAsioTlsStream(UnbufferedAsioTlsStreamParams& init) + : stream(init.IoService, init.SslContext), m_VerifyOK(true), m_Hostname(init.Hostname) { } + + bool IsVerifyOK() const; + String GetVerifyError() const; + + template + inline + auto async_handshake(handshake_type type, Args&&... args) -> decltype(Parent::async_handshake(type, std::forward(args)...)) + { + BeforeHandshake(type); + + return Parent::async_handshake(type, std::forward(args)...); + } + + template + inline + auto handshake(handshake_type type, Args&&... args) -> decltype(Parent::handshake(type, std::forward(args)...)) + { + BeforeHandshake(type); + + return Parent::handshake(type, std::forward(args)...); + } + +private: + bool m_VerifyOK; + String m_VerifyError; + String m_Hostname; + + void BeforeHandshake(handshake_type type); }; class AsioTlsStream : public boost::asio::buffered_stream { public: inline - AsioTlsStream(boost::asio::io_service& ioService, boost::asio::ssl::context& sslContext) - : AsioTlsStream(std::make_pair(&ioService, &sslContext)) + AsioTlsStream(boost::asio::io_service& ioService, boost::asio::ssl::context& sslContext, const String& hostname = String()) + : AsioTlsStream(UnbufferedAsioTlsStreamParams{ioService, sslContext, hostname}) { } private: inline - AsioTlsStream(std::pair init) + AsioTlsStream(UnbufferedAsioTlsStreamParams init) : buffered_stream(init) { } diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index f3057c289..cf4de5952 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -20,12 +20,11 @@ #include "base/context.hpp" #include "base/statsfunction.hpp" #include "base/exception.hpp" +#include "base/tcpsocket.hpp" #include #include #include #include -#include -#include #include #include #include @@ -462,34 +461,9 @@ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint) << "Reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'"; try { - auto sslConn (std::make_shared(io, *sslContext)); + auto sslConn (std::make_shared(io, *sslContext, endpoint->GetName())); - { - tcp::resolver resolver (io); - tcp::resolver::query query (host, port); - auto result (resolver.async_resolve(query, yc)); - auto current (result.begin()); - - for (;;) { - auto& tcpConn (sslConn->lowest_layer()); - - try { - tcpConn.open(current->endpoint().protocol()); - tcpConn.set_option(tcp::socket::keep_alive(true)); - tcpConn.async_connect(current->endpoint(), yc); - - break; - } catch (const std::exception&) { - if (++current == result.end()) { - throw; - } - - if (tcpConn.is_open()) { - tcpConn.close(); - } - } - } - } + Connect(sslConn->lowest_layer(), host, port, yc); NewClientHandler(yc, sslConn, endpoint->GetName(), RoleClient); @@ -551,39 +525,6 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const auto& sslConn (client->next_layer()); - sslConn.set_verify_mode(ssl::verify_peer | ssl::verify_client_once); - - bool verify_ok = true; - String verifyError; - - sslConn.set_verify_callback([&verify_ok, &verifyError](bool preverified, ssl::verify_context& ctx) { - if (!preverified) { - verify_ok = false; - - std::ostringstream msgbuf; - int err = X509_STORE_CTX_get_error(ctx.native_handle()); - - msgbuf << "code " << err << ": " << X509_verify_cert_error_string(err); - verifyError = msgbuf.str(); - } - - return true; - }); - - if (role == RoleClient) { - String environmentName = Application::GetAppEnvironment(); - String serverName = hostname; - - if (!environmentName.IsEmpty()) - serverName += ":" + environmentName; - -#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME - if (!hostname.IsEmpty()) { - SSL_set_tlsext_host_name(sslConn.native_handle(), serverName.CStr()); - } -#endif /* SSL_CTRL_SET_TLSEXT_HOSTNAME */ - } - try { sslConn.async_handshake(role == RoleClient ? sslConn.client : sslConn.server, yc); } catch (const std::exception& ex) { @@ -601,10 +542,15 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const }); std::shared_ptr cert (SSL_get_peer_certificate(sslConn.native_handle()), X509_free); + bool verify_ok = false; String identity; Endpoint::Ptr endpoint; if (cert) { + verify_ok = sslConn.IsVerifyOK(); + + String verifyError = sslConn.GetVerifyError(); + try { identity = GetCertificateCN(cert); } catch (const std::exception&) { @@ -640,8 +586,6 @@ void ApiListener::NewClientHandlerInternal(boost::asio::yield_context yc, const log << " (no Endpoint object found for identity)"; } } else { - verify_ok = false; - Log(LogInformation, "ApiListener") << "New client connection " << conninfo << " (no client certificate)"; } From 79e95d2355acf4c781e171fbbc9467b61e14f21f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 26 Feb 2019 10:17:10 +0100 Subject: [PATCH 52/67] Introduce JsonRpcConnection#SendMessageInternal() --- lib/remote/jsonrpcconnection-heartbeat.cpp | 4 +--- lib/remote/jsonrpcconnection.cpp | 14 ++++++++------ lib/remote/jsonrpcconnection.hpp | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index da6afe4b7..f2e7f4045 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -36,15 +36,13 @@ void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) break; } - m_OutgoingMessagesQueue.emplace_back(new Dictionary({ + SendMessageInternal(new Dictionary({ { "jsonrpc", "2.0" }, { "method", "event::Heartbeat" }, { "params", new Dictionary({ { "timeout", 120 } }) } })); - - m_OutgoingMessagesQueued.Set(); } } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index a0f37f950..16066a6ae 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -160,10 +160,13 @@ ConnectionRole JsonRpcConnection::GetRole() const void JsonRpcConnection::SendMessage(const Dictionary::Ptr& message) { - m_IoStrand.post([this, message]() { - m_OutgoingMessagesQueue.emplace_back(message); - m_OutgoingMessagesQueued.Set(); - }); + m_IoStrand.post([this, message]() { SendMessageInternal(message); }); +} + +void JsonRpcConnection::SendMessageInternal(const Dictionary::Ptr& message) +{ + m_OutgoingMessagesQueue.emplace_back(message); + m_OutgoingMessagesQueued.Set(); } void JsonRpcConnection::Disconnect() @@ -277,8 +280,7 @@ void JsonRpcConnection::MessageHandler(const String& jsonString) resultMessage->Set("jsonrpc", "2.0"); resultMessage->Set("id", message->Get("id")); - m_OutgoingMessagesQueue.emplace_back(resultMessage); - m_OutgoingMessagesQueued.Set(); + SendMessageInternal(resultMessage); } } diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index b0679d368..8f48fc4cd 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -86,6 +86,8 @@ private: void MessageHandler(const String& jsonString); void CertificateRequestResponseHandler(const Dictionary::Ptr& message); + + void SendMessageInternal(const Dictionary::Ptr& request); }; } From 5208448b765c3114ad70e4f45587920e815d0bea Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 26 Feb 2019 11:13:34 +0100 Subject: [PATCH 53/67] Restore the previous performance of replaying logs --- lib/remote/apilistener.cpp | 2 +- lib/remote/jsonrpc.cpp | 12 +++++++++++- lib/remote/jsonrpc.hpp | 1 + lib/remote/jsonrpcconnection.cpp | 13 +++++++++++-- lib/remote/jsonrpcconnection.hpp | 3 ++- 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index cf4de5952..75a676143 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -1277,7 +1277,7 @@ void ApiListener::ReplayLog(const JsonRpcConnection::Ptr& client) } try { - client->SendMessage(JsonDecode(pmessage->Get("message"))); + client->SendRawMessage(pmessage->Get("message")); count++; } catch (const std::exception& ex) { Log(LogWarning, "ApiListener") diff --git a/lib/remote/jsonrpc.cpp b/lib/remote/jsonrpc.cpp index 5f4b6ae91..03f3c7d0e 100644 --- a/lib/remote/jsonrpc.cpp +++ b/lib/remote/jsonrpc.cpp @@ -68,8 +68,18 @@ size_t JsonRpc::SendMessage(const Stream::Ptr& stream, const Dictionary::Ptr& me */ size_t JsonRpc::SendMessage(const std::shared_ptr& stream, const Dictionary::Ptr& message, boost::asio::yield_context yc) { - String json = JsonEncode(message); + return JsonRpc::SendRawMessage(stream, JsonEncode(message), yc); +} +/** + * Sends a message to the connected peer and returns the bytes sent. + * + * @param message The message. + * + * @return The amount of bytes sent. + */ +size_t JsonRpc::SendRawMessage(const std::shared_ptr& stream, const String& json, boost::asio::yield_context yc) +{ #ifdef I2_DEBUG if (GetDebugJsonRpcCached()) std::cerr << ConsoleColorTag(Console_ForegroundBlue) << ">> " << json << ConsoleColorTag(Console_Normal) << "\n"; diff --git a/lib/remote/jsonrpc.hpp b/lib/remote/jsonrpc.hpp index 137d42a3b..faf9c07e8 100644 --- a/lib/remote/jsonrpc.hpp +++ b/lib/remote/jsonrpc.hpp @@ -23,6 +23,7 @@ class JsonRpc public: static size_t SendMessage(const Stream::Ptr& stream, const Dictionary::Ptr& message); static size_t SendMessage(const std::shared_ptr& stream, const Dictionary::Ptr& message, boost::asio::yield_context yc); + static size_t SendRawMessage(const std::shared_ptr& stream, const String& json, boost::asio::yield_context yc); static StreamReadStatus ReadMessage(const Stream::Ptr& stream, String *message, StreamReadContext& src, bool may_wait = false, ssize_t maxMessageLength = -1); static String ReadMessage(const std::shared_ptr& stream, boost::asio::yield_context yc, ssize_t maxMessageLength = -1); static Dictionary::Ptr DecodeMessage(const String& message); diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 16066a6ae..e54a998d0 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -6,6 +6,7 @@ #include "remote/jsonrpc.hpp" #include "base/configtype.hpp" #include "base/io-engine.hpp" +#include "base/json.hpp" #include "base/objectlock.hpp" #include "base/utility.hpp" #include "base/logger.hpp" @@ -106,7 +107,7 @@ void JsonRpcConnection::WriteOutgoingMessages(boost::asio::yield_context yc) if (!queue.empty()) { try { for (auto& message : queue) { - size_t bytesSent = JsonRpc::SendMessage(m_Stream, message, yc); + size_t bytesSent = JsonRpc::SendRawMessage(m_Stream, message, yc); if (m_Endpoint) { m_Endpoint->AddMessageSent(bytesSent); @@ -163,9 +164,17 @@ void JsonRpcConnection::SendMessage(const Dictionary::Ptr& message) m_IoStrand.post([this, message]() { SendMessageInternal(message); }); } +void JsonRpcConnection::SendRawMessage(const String& message) +{ + m_IoStrand.post([this, message]() { + m_OutgoingMessagesQueue.emplace_back(message); + m_OutgoingMessagesQueued.Set(); + }); +} + void JsonRpcConnection::SendMessageInternal(const Dictionary::Ptr& message) { - m_OutgoingMessagesQueue.emplace_back(message); + m_OutgoingMessagesQueue.emplace_back(JsonEncode(message)); m_OutgoingMessagesQueued.Set(); } diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 8f48fc4cd..994dd7368 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -55,6 +55,7 @@ public: void Disconnect(); void SendMessage(const Dictionary::Ptr& request); + void SendRawMessage(const String& request); static Value HeartbeatAPIHandler(const intrusive_ptr& origin, const Dictionary::Ptr& params); @@ -72,7 +73,7 @@ private: double m_Seen; double m_NextHeartbeat; boost::asio::io_service::strand m_IoStrand; - std::vector m_OutgoingMessagesQueue; + std::vector m_OutgoingMessagesQueue; AsioConditionVariable m_OutgoingMessagesQueued; AsioConditionVariable m_WriterDone; bool m_ShuttingDown; From 915525dbcdb1af544aa7518250955d9c1d78fa3a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 7 Mar 2019 15:38:25 +0100 Subject: [PATCH 54/67] Doc: adjust default of ApiListener#bind_host --- doc/09-object-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index f3f69976f..e79861588 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -57,7 +57,7 @@ Configuration Attributes: ca\_path | String | **Deprecated.** Path to the CA certificate file. ticket\_salt | String | **Optional.** Private key for [CSR auto-signing](06-distributed-monitoring.md#distributed-monitoring-setup-csr-auto-signing). **Required** for a signing master instance. crl\_path | String | **Optional.** Path to the CRL file. - bind\_host | String | **Optional.** The IP address the api listener should be bound to. Defaults to `0.0.0.0`. + bind\_host | String | **Optional.** The IP address the api listener should be bound to. If not specified, the ApiListener is bound to `::` and listens for both IPv4 and IPv6 connections. bind\_port | Number | **Optional.** The port the api listener should be bound to. Defaults to `5665`. accept\_config | Boolean | **Optional.** Accept zone configuration. Defaults to `false`. accept\_commands | Boolean | **Optional.** Accept remote commands. Defaults to `false`. From 5b2c1f023da4184aa6b7896363a2544ac5a0b8a5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 11 Mar 2019 10:12:05 +0100 Subject: [PATCH 55/67] Rename preventGc to keepAlive --- lib/remote/httpserverconnection.cpp | 10 +++++----- lib/remote/jsonrpcconnection.cpp | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index af2a69b42..2d187faf5 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -51,19 +51,19 @@ void HttpServerConnection::Start() { namespace asio = boost::asio; - HttpServerConnection::Ptr preventGc (this); + HttpServerConnection::Ptr keepAlive (this); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { ProcessMessages(yc); }); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { CheckLiveness(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { ProcessMessages(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { CheckLiveness(yc); }); } void HttpServerConnection::Disconnect() { namespace asio = boost::asio; - HttpServerConnection::Ptr preventGc (this); + HttpServerConnection::Ptr keepAlive (this); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { if (!m_ShuttingDown) { m_ShuttingDown = true; diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index e54a998d0..9019f425c 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -41,12 +41,12 @@ void JsonRpcConnection::Start() { namespace asio = boost::asio; - JsonRpcConnection::Ptr preventGc (this); + JsonRpcConnection::Ptr keepAlive (this); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleIncomingMessages(yc); }); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { WriteOutgoingMessages(yc); }); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { HandleAndWriteHeartbeats(yc); }); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { CheckLiveness(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { HandleIncomingMessages(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { WriteOutgoingMessages(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { HandleAndWriteHeartbeats(yc); }); + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { CheckLiveness(yc); }); } void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) @@ -182,9 +182,9 @@ void JsonRpcConnection::Disconnect() { namespace asio = boost::asio; - JsonRpcConnection::Ptr preventGc (this); + JsonRpcConnection::Ptr keepAlive (this); - asio::spawn(m_IoStrand, [this, preventGc](asio::yield_context yc) { + asio::spawn(m_IoStrand, [this, keepAlive](asio::yield_context yc) { if (!m_ShuttingDown) { m_ShuttingDown = true; From f4193e40a1016e28ffcbbf8a4db067019e9eb239 Mon Sep 17 00:00:00 2001 From: Markus Frosch Date: Mon, 11 Mar 2019 16:14:41 +0100 Subject: [PATCH 56/67] appveyor: Use Boost 1.66 --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9d1150f72..48fb507b0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,8 +7,9 @@ platform: x64 environment: CMAKE_GENERATOR: "Visual Studio 15 2017 Win64" VSCMD_VER: 15.0 - BOOST_ROOT: 'C:\Libraries\boost_1_65_1' - BOOST_LIBRARYDIR: 'C:\Libraries\boost_1_65_1\lib64-msvc-14.1' + # https://www.appveyor.com/docs/windows-images-software/#boost + BOOST_ROOT: 'C:\Libraries\boost_1_66_0' + BOOST_LIBRARYDIR: 'C:\Libraries\boost_1_66_0\lib64-msvc-14.1' BISON_BINARY: 'C:\ProgramData\chocolatey\lib\winflexbison3\tools\win_bison.exe' FLEX_BINARY: 'C:\ProgramData\chocolatey\lib\winflexbison3\tools\win_flex.exe' CMAKE_BUILD_TYPE: Debug From 7ec1e638a89f8da55a3c77ef418aab64361ac8f4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Mar 2019 11:36:30 +0100 Subject: [PATCH 57/67] Turn shortcut UnbufferedAsioTlsStream::Parent into a base class --- lib/base/tlsstream.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index 2432d16e8..5f5fc6d73 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -106,11 +106,10 @@ struct UnbufferedAsioTlsStreamParams const String& Hostname; }; -class UnbufferedAsioTlsStream : public boost::asio::ssl::stream -{ -private: - typedef boost::asio::ssl::stream Parent; +typedef boost::asio::ssl::stream AsioTcpTlsStream; +class UnbufferedAsioTlsStream : public AsioTcpTlsStream +{ public: inline UnbufferedAsioTlsStream(UnbufferedAsioTlsStreamParams& init) @@ -123,20 +122,20 @@ public: template inline - auto async_handshake(handshake_type type, Args&&... args) -> decltype(Parent::async_handshake(type, std::forward(args)...)) + auto async_handshake(handshake_type type, Args&&... args) -> decltype(AsioTcpTlsStream::async_handshake(type, std::forward(args)...)) { BeforeHandshake(type); - return Parent::async_handshake(type, std::forward(args)...); + return AsioTcpTlsStream::async_handshake(type, std::forward(args)...); } template inline - auto handshake(handshake_type type, Args&&... args) -> decltype(Parent::handshake(type, std::forward(args)...)) + auto handshake(handshake_type type, Args&&... args) -> decltype(AsioTcpTlsStream::handshake(type, std::forward(args)...)) { BeforeHandshake(type); - return Parent::handshake(type, std::forward(args)...); + return AsioTcpTlsStream::handshake(type, std::forward(args)...); } private: From bf23e5392b1f26aefccb7667797834e8a0759f7e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Mar 2019 11:47:53 +0100 Subject: [PATCH 58/67] UnbufferedAsioTlsStream: don't rely on *this in decltype()s for methods' return types --- lib/base/tlsstream.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index 5f5fc6d73..6156a3d2f 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -122,7 +122,7 @@ public: template inline - auto async_handshake(handshake_type type, Args&&... args) -> decltype(AsioTcpTlsStream::async_handshake(type, std::forward(args)...)) + auto async_handshake(handshake_type type, Args&&... args) -> decltype(((AsioTcpTlsStream*)nullptr)->async_handshake(type, std::forward(args)...)) { BeforeHandshake(type); @@ -131,7 +131,7 @@ public: template inline - auto handshake(handshake_type type, Args&&... args) -> decltype(AsioTcpTlsStream::handshake(type, std::forward(args)...)) + auto handshake(handshake_type type, Args&&... args) -> decltype(((AsioTcpTlsStream*)nullptr)->handshake(type, std::forward(args)...)) { BeforeHandshake(type); From ff3a2fe3dabc68e4b0711e6848aa50af7465576e Mon Sep 17 00:00:00 2001 From: Markus Frosch Date: Tue, 12 Mar 2019 15:53:52 +0100 Subject: [PATCH 59/67] CMake: Handle INSTALL_RPATH globally and allow external additions --- CMakeLists.txt | 1 + icinga-app/CMakeLists.txt | 1 - plugins/CMakeLists.txt | 3 --- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b53844fd..888b462a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,7 @@ if(WIN32) endif() set(CMAKE_MACOSX_RPATH 1) +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH};${CMAKE_INSTALL_FULL_LIBDIR}/icinga2") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Qunused-arguments -fcolor-diagnostics") diff --git a/icinga-app/CMakeLists.txt b/icinga-app/CMakeLists.txt index d3ecfe8ab..ee3443b28 100644 --- a/icinga-app/CMakeLists.txt +++ b/icinga-app/CMakeLists.txt @@ -68,7 +68,6 @@ target_link_libraries(icinga-app ${base_DEPS}) set_target_properties ( icinga-app PROPERTIES - INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 FOLDER Bin OUTPUT_NAME icinga2 ) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index ebf0c77df..27fddecff 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -9,7 +9,6 @@ add_executable(check_nscp_api target_link_libraries(check_nscp_api ${base_DEPS}) set_target_properties ( check_nscp_api PROPERTIES - INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 DEFINE_SYMBOL I2_PLUGINS_BUILD FOLDER Plugins) @@ -32,7 +31,6 @@ if (WIN32) set_target_properties( thresholds PROPERTIES - INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 FOLDER Plugins ) @@ -50,7 +48,6 @@ if (WIN32) set_target_properties( ${check_OUT} PROPERTIES - INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 DEFINE_SYMBOL I2_PLUGINS_BUILD FOLDER Plugins ) From d428bdf384540f59b2e0094dcf5ba812cc2cc8a1 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 12 Mar 2019 17:55:39 +0100 Subject: [PATCH 60/67] Add missing includes --- lib/base/tlsstream.cpp | 1 + lib/remote/jsonrpcconnection.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/base/tlsstream.cpp b/lib/base/tlsstream.cpp index 1693b9edc..38913f28a 100644 --- a/lib/base/tlsstream.cpp +++ b/lib/base/tlsstream.cpp @@ -1,5 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ +#include "base/application.hpp" #include "base/tlsstream.hpp" #include "base/utility.hpp" #include "base/exception.hpp" diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 9019f425c..b850b7884 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -4,6 +4,7 @@ #include "remote/apilistener.hpp" #include "remote/apifunction.hpp" #include "remote/jsonrpc.hpp" +#include "base/defer.hpp" #include "base/configtype.hpp" #include "base/io-engine.hpp" #include "base/json.hpp" From 24c9542b5b84446a5678f3704e9c16a7f5a22f20 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 1 Apr 2019 10:14:24 +0200 Subject: [PATCH 61/67] HttpServerConnection: fix side effect of HTTP parser's default body limit --- lib/remote/httpserverconnection.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index 2d187faf5..d041e23ac 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -406,6 +406,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) http::response response; parser.header_limit(1024 * 1024); + parser.body_limit(-1); response.set(http::field::server, l_ServerHeader); From 3a6caa2800f8c54a548a6ec370c9339e61044541 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 1 Apr 2019 12:43:38 +0200 Subject: [PATCH 62/67] Respect Accept:application/json where possible --- lib/remote/httpserverconnection.cpp | 34 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index d041e23ac..af5169952 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -102,6 +102,8 @@ bool EnsureValidHeaders( { namespace http = boost::beast::http; + bool httpError = true; + try { try { http::async_read_header(stream, buf, parser, yc); @@ -115,6 +117,8 @@ bool EnsureValidHeaders( throw std::invalid_argument(ex.what()); } + httpError = false; + switch (parser.get().version()) { case 10: case 11: @@ -124,9 +128,18 @@ bool EnsureValidHeaders( } } catch (const std::invalid_argument& ex) { response.result(http::status::bad_request); - response.set(http::field::content_type, "text/html"); - response.body() = String("

Bad Request

") + ex.what() + "

"; - response.set(http::field::content_length, response.body().size()); + + if (!httpError && parser.get()[http::field::accept] == "application/json") { + HttpUtility::SendJsonBody(response, nullptr, new Dictionary({ + { "error", 400 }, + { "status", String("Bad Request: ") + ex.what() } + })); + } else { + response.set(http::field::content_type, "text/html"); + response.body() = String("

Bad Request

") + ex.what() + "

"; + response.set(http::field::content_length, response.body().size()); + } + response.set(http::field::connection, "close"); http::async_write(stream, response, yc); @@ -333,9 +346,18 @@ bool EnsureValidBody( */ response.result(http::status::bad_request); - response.set(http::field::content_type, "text/html"); - response.body() = String("

Bad Request

") + ex.what() + "

"; - response.set(http::field::content_length, response.body().size()); + + if (parser.get()[http::field::accept] == "application/json") { + HttpUtility::SendJsonBody(response, nullptr, new Dictionary({ + { "error", 400 }, + { "status", String("Bad Request: ") + ex.what() } + })); + } else { + response.set(http::field::content_type, "text/html"); + response.body() = String("

Bad Request

") + ex.what() + "

"; + response.set(http::field::content_length, response.body().size()); + } + response.set(http::field::connection, "close"); http::async_write(stream, response, yc); From 64b2ac4b3031d076671849cf0345541fae82d429 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 1 Apr 2019 15:06:17 +0200 Subject: [PATCH 63/67] ApiListener: drop unused thread pool --- lib/remote/apilistener.cpp | 16 ---------------- lib/remote/apilistener.hpp | 3 --- 2 files changed, 19 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 75a676143..6da42b68c 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -109,22 +109,6 @@ void ApiListener::CopyCertificateFile(const String& oldCertPath, const String& n } } -/** - * Returns the API thread pool. - * - * @returns The API thread pool. - */ -ThreadPool& ApiListener::GetTP() -{ - static ThreadPool tp; - return tp; -} - -void ApiListener::EnqueueAsyncCallback(const std::function& callback, SchedulerPolicy policy) -{ - GetTP().Post(callback, policy); -} - void ApiListener::OnConfigLoaded() { if (m_Instance) diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index b88f1de1e..7072a157f 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -134,9 +134,6 @@ private: void NewClientHandlerInternal(boost::asio::yield_context yc, const std::shared_ptr& client, const String& hostname, ConnectionRole role); void ListenerCoroutineProc(boost::asio::yield_context yc, const std::shared_ptr& server, const std::shared_ptr& sslContext); - static ThreadPool& GetTP(); - static void EnqueueAsyncCallback(const std::function& callback, SchedulerPolicy policy = DefaultScheduler); - WorkQueue m_RelayQueue; WorkQueue m_SyncQueue{0, 4}; From 0e5d0e0077010d47500d9d99c4357cf83fd72bc5 Mon Sep 17 00:00:00 2001 From: Markus Frosch Date: Mon, 1 Apr 2019 15:22:41 +0200 Subject: [PATCH 64/67] travis: Switch to xenial and newer Boost --- .travis.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23405135f..eaf8ca824 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial sudo: false language: cpp @@ -15,16 +15,18 @@ before_install: - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- addons: - apt_packages: - - libboost-all-dev - - flex - - bison - - libssl-dev - - libpq-dev - - libmysqlclient-dev - - libedit-dev - - libwxbase3.0-dev - - libwxgtk3.0-dev + apt: + sources: + - sourceline: 'deb http://packages.icinga.com/ubuntu icinga-xenial main' + key_url: 'https://packages.icinga.com/icinga.key' + packages: + - libboost1.67-icinga-all-dev + - flex + - bison + - libssl-dev + - libpq-dev + - libmysqlclient-dev + - libedit-dev coverity_scan: project: name: "Icinga/icinga2" From e4a3b6425877b39bc97d0fdbfb970d298b74caa6 Mon Sep 17 00:00:00 2001 From: Markus Frosch Date: Mon, 1 Apr 2019 15:25:19 +0200 Subject: [PATCH 65/67] travis: Remove converity --- .travis.yml | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index eaf8ca824..337bb900e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,15 +5,6 @@ language: cpp cache: ccache -env: - global: - # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created - # via the "travis encrypt" command using the project repo's public key - - secure: "eOnFdiRhB7VUZY7Of4Ff0px93HRWGcD4fXCPiy8V2OC2ER98CYCVw7PKt2Is6i/yTveFTps1kObOo0T03aUT8y/xeBy/wMuJYk1d6mVgmSXOjxcxjQVTUh4J+xB+k/R6FoP2dirNDbvSayCj9Fi9toN9hQHMM8oAZOZfiKmYTJc=" - -before_install: - - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- - addons: apt: sources: @@ -27,26 +18,14 @@ addons: - libpq-dev - libmysqlclient-dev - libedit-dev - coverity_scan: - project: - name: "Icinga/icinga2" - notification_email: icinga2@icinga.com - build_command_prepend: "cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/tmp/icinga2 -DICINGA2_PLUGINDIR=/tmp/icinga2/sbin -DICINGA2_UNITY_BUILD=ON" - build_command: "make -j 2" - branch_pattern: coverity_scan - before_script: - - if [ "$COVERITY_SCAN_BRANCH" != 1 ]; then - mkdir build && - cd build && - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/icinga2 -DICINGA2_PLUGINDIR=/tmp/icinga2/sbin; - fi + - mkdir build + - cd build + - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/icinga2 -DICINGA2_PLUGINDIR=/tmp/icinga2/sbin script: - - if [ "$COVERITY_SCAN_BRANCH" != 1 ]; then - make && - make test && - make install && - /tmp/icinga2/sbin/icinga2 --version && - /tmp/icinga2/sbin/icinga2 daemon -C -DRunAsUser=$(id -u -n) -DRunAsGroup=$(id -g -n); - fi + - make + - make test + - make install + - /tmp/icinga2/sbin/icinga2 --version + - /tmp/icinga2/sbin/icinga2 daemon -C -DRunAsUser=$(id -u -n) -DRunAsGroup=$(id -g -n) From ebf9f9d9f598341d29ada6e36a14f510560da307 Mon Sep 17 00:00:00 2001 From: Markus Frosch Date: Mon, 1 Apr 2019 15:31:44 +0200 Subject: [PATCH 66/67] travis: Add boost path config And disable unity builds for travis. [skip appveyor] --- .travis.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 337bb900e..03f43f032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,9 +19,20 @@ addons: - libmysqlclient-dev - libedit-dev before_script: + - arch=$(uname -m) - mkdir build - cd build - - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/icinga2 -DICINGA2_PLUGINDIR=/tmp/icinga2/sbin + - > + cmake .. + -DCMAKE_BUILD_TYPE=Debug + -DICINGA2_UNITY_BUILD=Off + -DCMAKE_INSTALL_PREFIX=/tmp/icinga2 + -DICINGA2_PLUGINDIR=/tmp/icinga2/sbin + -DBoost_NO_BOOST_CMAKE=TRUE + -DBoost_NO_SYSTEM_PATHS=TRUE + -DBOOST_LIBRARYDIR=/usr/lib/${arch}-linux-gnu/icinga-boost + -DBOOST_INCLUDEDIR=/usr/include/icinga-boost + -DCMAKE_INSTALL_RPATH=/usr/lib/${arch}-linux-gnu/icinga-boost script: - make From 5c2aaf638013e9b43de1ea2d31dee57feb23ae09 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 1 Apr 2019 16:13:37 +0200 Subject: [PATCH 67/67] Improve error logging on connection failure (cluster) --- lib/remote/apilistener.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 6da42b68c..de6e754c0 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -413,7 +413,8 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const std asio::spawn(io, [this, sslConn](asio::yield_context yc) { NewClientHandler(yc, sslConn, String(), RoleServer); }); } catch (const std::exception& ex) { - Log(LogCritical, "ApiListener") << "Cannot accept new connection: " << DiagnosticInformation(ex, false); + Log(LogCritical, "ApiListener") + << "Cannot accept new connection: " << DiagnosticInformation(ex, false); } } } @@ -457,11 +458,8 @@ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint) } catch (const std::exception& ex) { endpoint->SetConnecting(false); - std::ostringstream info; - info << "Cannot connect to host '" << host << "' on port '" << port << "'"; - Log(LogCritical, "ApiListener", info.str()); - Log(LogDebug, "ApiListener") - << info.str() << "\n" << DiagnosticInformation(ex); + Log(LogCritical, "ApiListener") + << "Cannot connect to host '" << host << "' on port '" << port << "': " << ex.what(); } }); }