tyler.durden/icinga2
1
0
Fork 0
mirror of https://github.com/Icinga/icinga2.git synced 2025-04-08 17:05:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6988 from Icinga/feature/thread-pool-6985

Browse Source 
ThreadPool: use the Boost ASIO thread pool under the hood
This commit is contained in:
Michael Friedrich 2019-04-25 14:20:42 +02:00 committed by GitHub
commit 4d05eccddf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 74 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/base/exception.cpp
View File

@ -1,6 +1,7 @@
/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
#include "base/exception.hpp"
#include <boost/thread/tss.hpp>
#ifdef HAVE_CXXABI_H
# include <cxxabi.h>

1
lib/base/exception.hpp
View File

@ -7,7 +7,6 @@
#include "base/string.hpp"
#include "base/stacktrace.hpp"
#include "base/context.hpp"
#include "base/utility.hpp"
#include "base/debuginfo.hpp"
#include "base/dictionary.hpp"
#include "base/configobject.hpp"

1
lib/base/scriptglobal.cpp
View File

@ -10,6 +10,7 @@
#include "base/objectlock.hpp"
#include "base/exception.hpp"
#include "base/namespace.hpp"
#include "base/utility.hpp"
#include <fstream>
using namespace icinga;

1
lib/base/socketevents-epoll.cpp
View File

@ -3,6 +3,7 @@
#include "base/socketevents.hpp"
#include "base/exception.hpp"
#include "base/logger.hpp"
#include "base/utility.hpp"
#include <boost/thread/once.hpp>
#include <map>
#ifdef __linux__

1
lib/base/socketevents-poll.cpp
View File

@ -3,6 +3,7 @@
#include "base/socketevents.hpp"
#include "base/exception.hpp"
#include "base/logger.hpp"
#include "base/utility.hpp"
#include <boost/thread/once.hpp>
#include <map>

1
lib/base/socketevents.cpp
View File

@ -5,6 +5,7 @@
#include "base/logger.hpp"
#include "base/application.hpp"
#include "base/scriptglobal.hpp"
#include "base/utility.hpp"
#include <boost/thread/once.hpp>
#include <map>
#ifdef __linux__

367
lib/base/threadpool.cpp
View File

@ -1,23 +1,13 @@
/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
#include "base/threadpool.hpp"
#include "base/logger.hpp"
#include "base/debug.hpp"
#include "base/utility.hpp"
#include "base/exception.hpp"
#include "base/application.hpp"
#include <iostream>
#include <boost/thread/locks.hpp>
using namespace icinga;
int ThreadPool::m_NextID = 1;
ThreadPool::ThreadPool(size_t max_threads)
: m_ID(m_NextID++), m_MaxThreads(max_threads)
ThreadPool::ThreadPool(size_t threads)
: m_Threads(threads)
{
if (m_MaxThreads != UINT_MAX && m_MaxThreads < sizeof(m_Queues) / sizeof(m_Queues[0]))
m_MaxThreads = sizeof(m_Queues) / sizeof(m_Queues[0]);
Start();
}
@ -28,354 +18,19 @@ ThreadPool::~ThreadPool()
void ThreadPool::Start()
{
if (!m_Stopped)
return;
boost::unique_lock<decltype(m_Mutex)> lock (m_Mutex);
m_Stopped = false;
for (auto& queue : m_Queues)
queue.SpawnWorker(m_ThreadGroup);
m_MgmtThread = std::thread(std::bind(&ThreadPool::ManagerThreadProc, this));
if (!m_Pool) {
m_Pool = decltype(m_Pool)(new boost::asio::thread_pool(m_Threads));
}
}
void ThreadPool::Stop()
{
if (m_Stopped)
return;
boost::unique_lock<decltype(m_Mutex)> lock (m_Mutex);
{
boost::mutex::scoped_lock lock(m_MgmtMutex);
m_Stopped = true;
m_MgmtCV.notify_all();
}
if (m_MgmtThread.joinable())
m_MgmtThread.join();
for (auto& queue : m_Queues) {
boost::mutex::scoped_lock lock(queue.Mutex);
queue.Stopped = true;
queue.CV.notify_all();
}
m_ThreadGroup.join_all();
m_ThreadGroup.~thread_group();
new (&m_ThreadGroup) boost::thread_group();
for (auto& queue : m_Queues)
queue.Stopped = false;
m_Stopped = true;
}
/**
* Waits for work items and processes them.
*/
void ThreadPool::WorkerThread::ThreadProc(Queue& queue)
{
std::ostringstream idbuf;
idbuf << "Q #" << &queue << " W #" << this;
Utility::SetThreadName(idbuf.str());
for (;;) {
WorkItem wi;
{
boost::mutex::scoped_lock lock(queue.Mutex);
UpdateUtilization(ThreadIdle);
while (queue.Items.empty() && !queue.Stopped && !Zombie) {
if (queue.Items.empty())
queue.CVStarved.notify_all();
queue.CV.wait(lock);
}
if (Zombie)
break;
if (queue.Items.empty() && queue.Stopped)
break;
wi = queue.Items.front();
queue.Items.pop_front();
UpdateUtilization(ThreadBusy);
}
double st = Utility::GetTime();
#ifdef I2_DEBUG
# ifdef RUSAGE_THREAD
struct rusage usage_start, usage_end;
(void) getrusage(RUSAGE_THREAD, &usage_start);
# endif /* RUSAGE_THREAD */
#endif /* I2_DEBUG */
try {
if (wi.Callback)
wi.Callback();
} catch (const std::exception& ex) {
Log(LogCritical, "ThreadPool")
<< "Exception thrown in event handler:\n"
<< DiagnosticInformation(ex);
} catch (...) {
Log(LogCritical, "ThreadPool", "Exception of unknown type thrown in event handler.");
}
double et = Utility::GetTime();
double latency = st - wi.Timestamp;
{
boost::mutex::scoped_lock lock(queue.Mutex);
queue.WaitTime += latency;
queue.ServiceTime += et - st;
queue.TaskCount++;
}
#ifdef I2_DEBUG
# ifdef RUSAGE_THREAD
(void) getrusage(RUSAGE_THREAD, &usage_end);
double duser = (usage_end.ru_utime.tv_sec - usage_start.ru_utime.tv_sec) +
(usage_end.ru_utime.tv_usec - usage_start.ru_utime.tv_usec) / 1000000.0;
double dsys = (usage_end.ru_stime.tv_sec - usage_start.ru_stime.tv_sec) +
(usage_end.ru_stime.tv_usec - usage_start.ru_stime.tv_usec) / 1000000.0;
double dwait = (et - st) - (duser + dsys);
int dminfaults = usage_end.ru_minflt - usage_start.ru_minflt;
int dmajfaults = usage_end.ru_majflt - usage_start.ru_majflt;
int dvctx = usage_end.ru_nvcsw - usage_start.ru_nvcsw;
int divctx = usage_end.ru_nivcsw - usage_start.ru_nivcsw;
# endif /* RUSAGE_THREAD */
if (et - st > 0.5) {
Log(LogWarning, "ThreadPool")
# ifdef RUSAGE_THREAD
<< "Event call took user:" << duser << "s, system:" << dsys << "s, wait:" << dwait << "s, minor_faults:" << dminfaults << ", major_faults:" << dmajfaults << ", voluntary_csw:" << dvctx << ", involuntary_csw:" << divctx;
# else
<< "Event call took " << (et - st) << "s";
# endif /* RUSAGE_THREAD */
}
#endif /* I2_DEBUG */
}
boost::mutex::scoped_lock lock(queue.Mutex);
UpdateUtilization(ThreadDead);
Zombie = false;
}
/**
* Appends a work item to the work queue. Work items will be processed in FIFO order.
*
* @param callback The callback function for the work item.
* @param policy The scheduling policy
* @returns true if the item was queued, false otherwise.
*/
bool ThreadPool::Post(const ThreadPool::WorkFunction& callback, SchedulerPolicy policy)
{
WorkItem wi;
wi.Callback = callback;
wi.Timestamp = Utility::GetTime();
Queue& queue = m_Queues[Utility::Random() % (sizeof(m_Queues) / sizeof(m_Queues[0]))];
{
boost::mutex::scoped_lock lock(queue.Mutex);
if (queue.Stopped)
return false;
if (policy == LowLatencyScheduler)
queue.SpawnWorker(m_ThreadGroup);
queue.Items.emplace_back(std::move(wi));
queue.CV.notify_one();
}
return true;
}
void ThreadPool::ManagerThreadProc()
{
std::ostringstream idbuf;
idbuf << "TP #" << m_ID << " Manager";
Utility::SetThreadName(idbuf.str());
double lastStats = 0;
for (;;) {
size_t total_pending = 0, total_alive = 0;
double total_avg_latency = 0;
double total_utilization = 0;
{
boost::mutex::scoped_lock lock(m_MgmtMutex);
if (!m_Stopped)
m_MgmtCV.timed_wait(lock, boost::posix_time::milliseconds(500));
if (m_Stopped)
break;
}
for (auto& queue : m_Queues) {
size_t pending, alive = 0;
double avg_latency;
double utilization = 0;
boost::mutex::scoped_lock lock(queue.Mutex);
for (auto& thread : queue.Threads)
thread.UpdateUtilization();
pending = queue.Items.size();
for (auto& thread : queue.Threads) {
if (thread.State != ThreadDead && !thread.Zombie) {
alive++;
utilization += thread.Utilization * 100;
}
}
utilization /= alive;
if (queue.TaskCount > 0)
avg_latency = queue.WaitTime / (queue.TaskCount * 1.0);
else
avg_latency = 0;
if (utilization < 60 || utilization > 80 || alive < 8) {
double wthreads = std::ceil((utilization * alive) / 80.0);
int tthreads = wthreads - alive;
/* Make sure there is at least one thread per queue */
if (alive + tthreads < 1)
tthreads = 1 - alive;
/* Don't kill more than 2 threads at once. */
if (tthreads < -2)
tthreads = -2;
/* Spawn more workers if there are outstanding work items. */
if (tthreads > 0 && pending > 0)
tthreads = 2;
if (m_MaxThreads != UINT_MAX && (alive + tthreads) * (sizeof(m_Queues) / sizeof(m_Queues[0])) > m_MaxThreads)
tthreads = m_MaxThreads / (sizeof(m_Queues) / sizeof(m_Queues[0])) - alive;
if (tthreads != 0) {
Log(LogNotice, "ThreadPool")
<< "Thread pool; current: " << alive << "; adjustment: " << tthreads;
}
for (int i = 0; i < -tthreads; i++)
queue.KillWorker(m_ThreadGroup);
for (int i = 0; i < tthreads; i++)
queue.SpawnWorker(m_ThreadGroup);
}
queue.WaitTime = 0;
queue.ServiceTime = 0;
queue.TaskCount = 0;
total_pending += pending;
total_alive += alive;
total_avg_latency += avg_latency;
total_utilization += utilization;
}
double now = Utility::GetTime();
if (lastStats < now - 15) {
lastStats = now;
Log(LogNotice, "ThreadPool")
<< "Pool #" << m_ID << ": Pending tasks: " << total_pending << "; Average latency: "
<< (long)(total_avg_latency * 1000 / (sizeof(m_Queues) / sizeof(m_Queues[0]))) << "ms"
<< "; Threads: " << total_alive
<< "; Pool utilization: " << (total_utilization / (sizeof(m_Queues) / sizeof(m_Queues[0]))) << "%";
}
if (m_Pool) {
m_Pool->join();
m_Pool = nullptr;
}
}
/**
* Note: Caller must hold m_Mutex
*/
void ThreadPool::Queue::SpawnWorker(boost::thread_group& group)
{
for (auto& thread : Threads) {
if (thread.State == ThreadDead) {
Log(LogDebug, "ThreadPool", "Spawning worker thread.");
thread = WorkerThread(ThreadIdle);
thread.Thread = group.create_thread(std::bind(&ThreadPool::WorkerThread::ThreadProc, std::ref(thread), std::ref(*this)));
break;
}
}
}
/**
* Note: Caller must hold Mutex.
*/
void ThreadPool::Queue::KillWorker(boost::thread_group& group)
{
for (auto& thread : Threads) {
if (thread.State == ThreadIdle && !thread.Zombie) {
Log(LogDebug, "ThreadPool", "Killing worker thread.");
group.remove_thread(thread.Thread);
thread.Thread->detach();
delete thread.Thread;
thread.Zombie = true;
CV.notify_all();
break;
}
}
}
/**
* Note: Caller must hold queue Mutex.
*/
void ThreadPool::WorkerThread::UpdateUtilization(ThreadState state)
{
double utilization;
switch (State) {
case ThreadDead:
return;
case ThreadIdle:
utilization = 0;
break;
case ThreadBusy:
utilization = 1;
break;
default:
VERIFY(0);
}
double now = Utility::GetTime();
double time = now - LastUpdate;
const double avg_time = 5.0;
if (time > avg_time)
time = avg_time;
Utilization = (Utilization * (avg_time - time) + utilization * time) / avg_time;
LastUpdate = now;
if (state != ThreadUnspecified)
State = state;
}

122
lib/base/threadpool.hpp
View File

@ -3,18 +3,21 @@
#ifndef THREADPOOL_H
#define THREADPOOL_H
#include "base/i2-base.hpp"
#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition_variable.hpp>
#include <deque>
#include "base/exception.hpp"
#include "base/logger.hpp"
#include <cstddef>
#include <exception>
#include <functional>
#include <memory>
#include <thread>
#include <boost/asio/post.hpp>
#include <boost/asio/thread_pool.hpp>
#include <boost/thread/locks.hpp>
#include <boost/thread/shared_mutex.hpp>
namespace icinga
{
#define QUEUECOUNT 4U
enum SchedulerPolicy
{
DefaultScheduler,
@ -31,85 +34,48 @@ class ThreadPool
public:
typedef std::function<void ()> WorkFunction;
ThreadPool(size_t max_threads = UINT_MAX);
ThreadPool(size_t threads = std::thread::hardware_concurrency() * 2u);
~ThreadPool();
void Start();
void Stop();
bool Post(const WorkFunction& callback, SchedulerPolicy policy = DefaultScheduler);
/**
* Appends a work item to the work queue. Work items will be processed in FIFO order.
*
* @param callback The callback function for the work item.
* @returns true if the item was queued, false otherwise.
*/
template<class T>
bool Post(T callback, SchedulerPolicy)
{
boost::shared_lock<decltype(m_Mutex)> lock (m_Mutex);
if (m_Pool) {
boost::asio::post(*m_Pool, [callback]() {
try {
callback();
} catch (const std::exception& ex) {
Log(LogCritical, "ThreadPool")
<< "Exception thrown in event handler:\n"
<< DiagnosticInformation(ex);
} catch (...) {
Log(LogCritical, "ThreadPool", "Exception of unknown type thrown in event handler.");
}
});
return true;
} else {
return false;
}
}
private:
enum ThreadState
{
ThreadUnspecified,
ThreadDead,
ThreadIdle,
ThreadBusy
};
struct WorkItem
{
WorkFunction Callback;
double Timestamp;
};
struct Queue;
struct WorkerThread
{
ThreadState State{ThreadDead};
bool Zombie{false};
double Utilization{0};
double LastUpdate{0};
boost::thread *Thread{nullptr};
WorkerThread(ThreadState state = ThreadDead)
: State(state)
{ }
void UpdateUtilization(ThreadState state = ThreadUnspecified);
void ThreadProc(Queue& queue);
};
struct Queue
{
boost::mutex Mutex;
boost::condition_variable CV;
boost::condition_variable CVStarved;
std::deque<WorkItem> Items;
double WaitTime{0};
double ServiceTime{0};
int TaskCount{0};
bool Stopped{false};
WorkerThread Threads[16];
void SpawnWorker(boost::thread_group& group);
void KillWorker(boost::thread_group& group);
};
int m_ID;
static int m_NextID;
size_t m_MaxThreads;
boost::thread_group m_ThreadGroup;
std::thread m_MgmtThread;
boost::mutex m_MgmtMutex;
boost::condition_variable m_MgmtCV;
bool m_Stopped{true};
Queue m_Queues[QUEUECOUNT];
void ManagerThreadProc();
boost::shared_mutex m_Mutex;
std::unique_ptr<boost::asio::thread_pool> m_Pool;
size_t m_Threads;
};
}
#endif /* EVENTQUEUE_H */
#endif /* THREADPOOL_H */

1
lib/cli/apisetuputility.cpp
View File

@ -11,6 +11,7 @@
#include "base/tlsutility.hpp"
#include "base/scriptglobal.hpp"
#include "base/exception.hpp"
#include "base/utility.hpp"
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/case_conv.hpp>

1
lib/config/configcompilercontext.cpp
View File

@ -6,6 +6,7 @@
#include "base/netstring.hpp"
#include "base/exception.hpp"
#include "base/application.hpp"
#include "base/utility.hpp"
using namespace icinga;

1
lib/config/configitem.cpp
View File

@ -19,6 +19,7 @@
#include "base/json.hpp"
#include "base/exception.hpp"
#include "base/function.hpp"
#include "base/utility.hpp"
#include <boost/algorithm/string/join.hpp>
#include <sstream>
#include <fstream>

1
lib/remote/apilistener-filesync.cpp
View File

@ -7,6 +7,7 @@
#include "base/logger.hpp"
#include "base/convert.hpp"
#include "base/exception.hpp"
#include "base/utility.hpp"
#include <fstream>
#include <iomanip>

1
lib/remote/configfileshandler.cpp
View File

@ -5,6 +5,7 @@
#include "remote/httputility.hpp"
#include "remote/filterutility.hpp"
#include "base/exception.hpp"
#include "base/utility.hpp"
#include <boost/algorithm/string/join.hpp>
#include <fstream>

1
lib/remote/configobjectutility.cpp
View File

@ -8,6 +8,7 @@
#include "base/configwriter.hpp"
#include "base/exception.hpp"
#include "base/dependencygraph.hpp"
#include "base/utility.hpp"
#include <boost/algorithm/string/case_conv.hpp>
#include <fstream>

1
lib/remote/consolehandler.cpp
View File

@ -12,6 +12,7 @@
#include "base/timer.hpp"
#include "base/namespace.hpp"
#include "base/initialize.hpp"
#include "base/utility.hpp"
#include <boost/thread/once.hpp>
#include <set>

1
lib/remote/filterutility.cpp
View File

@ -8,6 +8,7 @@
#include "base/json.hpp"
#include "base/configtype.hpp"
#include "base/logger.hpp"
#include "base/utility.hpp"
#include <boost/algorithm/string/case_conv.hpp>
using namespace icinga;

9
plugins/check_nscp_api.cpp
View File

@ -2,14 +2,17 @@
#define VERSION "1.0.1"
#include "remote/httpclientconnection.hpp"
#include "remote/httprequest.hpp"
#include "remote/url-characters.hpp"
// ensure to include base first
#include "base/i2-base.hpp"
#include "base/application.hpp"
#include "base/json.hpp"
#include "base/string.hpp"
#include "base/logger.hpp"
#include "base/exception.hpp"
#include "base/utility.hpp"
#include "remote/httpclientconnection.hpp"
#include "remote/httprequest.hpp"
#include "remote/url-characters.hpp"
#include <boost/program_options.hpp>
#include <boost/algorithm/string/split.hpp>
#include <iostream>