tyler.durden
/
icinga2
mirror of https://github.com/Icinga/icinga2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7643 from Icinga/bugfix/icingadb-recover 

IcingaDB: handle Redis restarts
This commit is contained in:
Noah Hilverling 2019-11-26 16:05:04 +01:00 committed by GitHub
parent 6508e880b4 e522db20b1
commit 8a14f0e6f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
lib/icingadb/redisconnection.cpp
View File

@ -9,9 +9,9 @@
#include "base/objectlock.hpp"
#include "base/string.hpp"
#include "base/tcpsocket.hpp"
#include <boost/asio/post.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio.hpp>
#include <boost/coroutine/exceptions.hpp>
#include <boost/date_time/posix_time/posix_time_duration.hpp>
#include <boost/utility/string_view.hpp>
#include <boost/variant/get.hpp>
#include <exception>
@ -144,32 +144,42 @@ void RedisConnection::Connect(asio::yield_context& yc)
{
Defer notConnecting ([this]() { m_Connecting.store(m_Connected.load()); });
try {
if (m_Path.IsEmpty()) {
Log(LogInformation, "IcingaDB")
<< "Trying to connect to Redis server (async) on host '" << m_Host << ":" << m_Port << "'";
boost::asio::deadline_timer timer (m_Strand.context());
decltype(m_TcpConn) conn (new TcpConn(m_Strand.context()));
icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc);
m_TcpConn = std::move(conn);
} else {
Log(LogInformation, "IcingaDB")
<< "Trying to connect to Redis server (async) on unix socket path '" << m_Path << "'";
for (;;) {
try {
if (m_Path.IsEmpty()) {
Log(LogInformation, "IcingaDB")
<< "Trying to connect to Redis server (async) on host '" << m_Host << ":" << m_Port << "'";
decltype(m_UnixConn) conn (new UnixConn(m_Strand.context()));
conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc);
m_UnixConn = std::move(conn);
decltype(m_TcpConn) conn (new TcpConn(m_Strand.context()));
icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc);
m_TcpConn = std::move(conn);
} else {
Log(LogInformation, "IcingaDB")
<< "Trying to connect to Redis server (async) on unix socket path '" << m_Path << "'";
decltype(m_UnixConn) conn (new UnixConn(m_Strand.context()));
conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc);
m_UnixConn = std::move(conn);
}
m_Connected.store(true);
Log(LogInformation, "IcingaDB", "Connected to Redis server");
break;
} catch (const boost::coroutines::detail::forced_unwind&) {
throw;
} catch (const std::exception& ex) {
Log(LogCritical, "IcingaDB")
<< "Cannot connect to " << m_Host << ":" << m_Port << ": " << ex.what();
}
m_Connected.store(true);
Log(LogInformation, "IcingaDB", "Connected to Redis server");
} catch (const boost::coroutines::detail::forced_unwind&) {
throw;
} catch (const std::exception& ex) {
Log(LogCritical, "IcingaDB")
<< "Cannot connect to " << m_Host << ":" << m_Port << ": " << ex.what();
timer.expires_from_now(boost::posix_time::seconds(5));
timer.async_wait(yc);
}
}
void RedisConnection::ReadLoop(asio::yield_context& yc)

16
lib/icingadb/redisconnection.hpp
View File

@ -207,6 +207,8 @@ private:
template<class StreamPtr>
RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio::yield_context& yc)
{
namespace asio = boost::asio;
if (!stream) {
throw RedisDisconnected();
}
@ -221,6 +223,12 @@ RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio::
if (m_Connecting.exchange(false)) {
m_Connected.store(false);
stream = nullptr;
if (!m_Connecting.exchange(true)) {
Ptr keepAlive (this);
asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); });
}
}
throw;
@ -230,6 +238,8 @@ RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio::
template<class StreamPtr>
void RedisConnection::WriteOne(StreamPtr& stream, RedisConnection::Query& query, boost::asio::yield_context& yc)
{
namespace asio = boost::asio;
if (!stream) {
throw RedisDisconnected();
}
@ -245,6 +255,12 @@ void RedisConnection::WriteOne(StreamPtr& stream, RedisConnection::Query& query,
if (m_Connecting.exchange(false)) {
m_Connected.store(false);
stream = nullptr;
if (!m_Connecting.exchange(true)) {
Ptr keepAlive (this);
asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); });
}
}
throw;