icinga2/lib/base/threadpool.cpp

389 lines
9.8 KiB
C++
Raw Normal View History

2012-06-24 02:56:48 +02:00
/******************************************************************************
* Icinga 2 *
* Copyright (C) 2012 Icinga Development Team (http://www.icinga.org/) *
* *
* This program is free software; you can redistribute it and/or *
* modify it under the terms of the GNU General Public License *
* as published by the Free Software Foundation; either version 2 *
* of the License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the Free Software Foundation *
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. *
******************************************************************************/
2013-03-25 18:36:15 +01:00
#include "base/threadpool.h"
2013-03-16 21:18:53 +01:00
#include "base/logger_fwd.h"
#include "base/convert.h"
2013-08-28 08:18:58 +02:00
#include "base/debug.h"
2013-03-16 21:18:53 +01:00
#include "base/utility.h"
#include <sstream>
#include <iostream>
2013-03-15 18:21:29 +01:00
#include <boost/bind.hpp>
2013-03-16 21:18:53 +01:00
#include <boost/exception/diagnostic_information.hpp>
#include <boost/foreach.hpp>
2012-06-24 02:56:48 +02:00
using namespace icinga;
2013-03-25 18:36:15 +01:00
ThreadPool::ThreadPool(void)
2013-08-27 15:57:00 +02:00
: m_WaitTime(0), m_ServiceTime(0),
m_TaskCount(0), m_Stopped(false)
2013-02-18 14:40:24 +01:00
{
2013-03-23 15:57:12 +01:00
for (int i = 0; i < 2; i++)
2013-03-23 12:23:13 +01:00
SpawnWorker();
2013-02-26 10:13:54 +01:00
2013-04-19 12:58:16 +02:00
m_ManagerThread = boost::thread(boost::bind(&ThreadPool::ManagerThreadProc, this));
m_StatsThread = boost::thread(boost::bind(&ThreadPool::StatsThreadProc, this));
2013-02-18 14:40:24 +01:00
}
2012-07-13 23:33:30 +02:00
2013-03-25 18:36:15 +01:00
ThreadPool::~ThreadPool(void)
2013-02-15 06:47:26 +01:00
{
2013-02-17 19:14:34 +01:00
Stop();
2013-02-18 14:40:24 +01:00
Join();
2013-02-15 06:47:26 +01:00
}
2013-03-25 18:36:15 +01:00
void ThreadPool::Stop(void)
2013-02-15 06:47:26 +01:00
{
boost::mutex::scoped_lock lock(m_Mutex);
m_Stopped = true;
2013-04-19 12:58:16 +02:00
m_WorkCV.notify_all();
m_MgmtCV.notify_all();
2013-02-15 06:47:26 +01:00
}
2012-09-14 14:41:17 +02:00
/**
2013-02-18 14:40:24 +01:00
* Waits for all worker threads to finish.
2012-09-14 14:41:17 +02:00
*/
2013-03-25 18:36:15 +01:00
void ThreadPool::Join(void)
2012-06-24 02:56:48 +02:00
{
2013-03-23 12:23:13 +01:00
boost::mutex::scoped_lock lock(m_Mutex);
2013-03-25 18:36:15 +01:00
while (!m_Stopped || !m_WorkItems.empty()) {
2013-03-23 12:23:13 +01:00
lock.unlock();
Utility::Sleep(0.5);
lock.lock();
}
2013-04-19 12:58:16 +02:00
int alive;
do {
alive = 0;
for (size_t i = 0; i < sizeof(m_ThreadStats) / sizeof(m_ThreadStats[0]); i++) {
2013-04-19 12:58:16 +02:00
if (m_ThreadStats[i].State != ThreadDead) {
alive++;
KillWorker();
}
}
if (alive > 0) {
lock.unlock();
Utility::Sleep(0.5);
lock.lock();
}
} while (alive > 0);
m_ManagerThread.join();
m_StatsThread.join();
2013-02-17 19:14:34 +01:00
}
2012-06-24 02:56:48 +02:00
2013-02-17 19:14:34 +01:00
/**
2013-03-25 18:36:15 +01:00
* Waits for work items and processes them.
2013-02-17 19:14:34 +01:00
*/
2013-03-25 18:36:15 +01:00
void ThreadPool::QueueThreadProc(int tid)
2013-02-17 19:14:34 +01:00
{
2013-03-25 18:36:15 +01:00
std::ostringstream idbuf;
idbuf << "TP " << this << " Worker #" << tid;
Utility::SetThreadName(idbuf.str());
2013-02-19 23:02:08 +01:00
for (;;) {
2013-03-25 18:36:15 +01:00
WorkItem wi;
2012-06-24 02:56:48 +02:00
2013-02-17 19:14:34 +01:00
{
boost::mutex::scoped_lock lock(m_Mutex);
UpdateThreadUtilization(tid, ThreadIdle);
2013-08-27 15:57:00 +02:00
while (m_WorkItems.empty() && !m_Stopped && !m_ThreadStats[tid].Zombie)
2013-04-19 12:58:16 +02:00
m_WorkCV.wait(lock);
2013-02-17 19:14:34 +01:00
2013-08-27 15:57:00 +02:00
if (m_ThreadStats[tid].Zombie)
2013-03-23 12:23:13 +01:00
break;
2013-03-25 18:36:15 +01:00
if (m_WorkItems.empty() && m_Stopped)
2013-02-19 23:02:08 +01:00
break;
2013-03-25 18:36:15 +01:00
wi = m_WorkItems.front();
m_WorkItems.pop_front();
UpdateThreadUtilization(tid, ThreadBusy);
2013-02-17 19:14:34 +01:00
}
double st = Utility::GetTime();;
2013-03-06 11:03:50 +01:00
#ifdef _DEBUG
# ifdef RUSAGE_THREAD
struct rusage usage_start, usage_end;
2013-03-06 11:03:50 +01:00
(void) getrusage(RUSAGE_THREAD, &usage_start);
# endif /* RUSAGE_THREAD */
2013-03-06 11:03:50 +01:00
#endif /* _DEBUG */
2012-08-03 13:19:55 +02:00
try {
2013-03-25 18:36:15 +01:00
wi.Callback();
} catch (const std::exception& ex) {
2013-03-16 21:18:53 +01:00
std::ostringstream msgbuf;
msgbuf << "Exception thrown in event handler: " << std::endl
2013-03-16 21:18:53 +01:00
<< boost::diagnostic_information(ex);
2013-03-16 21:18:53 +01:00
Log(LogCritical, "base", msgbuf.str());
} catch (...) {
2013-03-16 21:18:53 +01:00
Log(LogCritical, "base", "Exception of unknown type thrown in event handler.");
}
2012-08-03 13:19:55 +02:00
double et = Utility::GetTime();
2013-03-25 18:36:15 +01:00
double latency = st - wi.Timestamp;
{
boost::mutex::scoped_lock lock(m_Mutex);
m_WaitTime += latency;
m_ServiceTime += et - st;
m_TaskCount++;
if (latency > m_MaxLatency)
m_MaxLatency = latency;
}
#ifdef _DEBUG
# ifdef RUSAGE_THREAD
(void) getrusage(RUSAGE_THREAD, &usage_end);
2013-03-06 11:03:50 +01:00
double duser = (usage_end.ru_utime.tv_sec - usage_start.ru_utime.tv_sec) +
(usage_end.ru_utime.tv_usec - usage_start.ru_utime.tv_usec) / 1000000.0;
2013-03-06 11:03:50 +01:00
double dsys = (usage_end.ru_stime.tv_sec - usage_start.ru_stime.tv_sec) +
(usage_end.ru_stime.tv_usec - usage_start.ru_stime.tv_usec) / 1000000.0;
2013-03-06 11:03:50 +01:00
double dwait = (et - st) - (duser + dsys);
2013-03-06 11:03:50 +01:00
int dminfaults = usage_end.ru_minflt - usage_start.ru_minflt;
int dmajfaults = usage_end.ru_majflt - usage_start.ru_majflt;
2013-03-06 11:03:50 +01:00
int dvctx = usage_end.ru_nvcsw - usage_start.ru_nvcsw;
int divctx = usage_end.ru_nivcsw - usage_start.ru_nivcsw;
# endif /* RUSAGE_THREAD */
if (et - st > 0.5) {
2013-03-18 22:40:40 +01:00
std::ostringstream msgbuf;
# ifdef RUSAGE_THREAD
msgbuf << "Event call took user:" << duser << "s, system:" << dsys << "s, wait:" << dwait << "s, minor_faults:" << dminfaults << ", major_faults:" << dmajfaults << ", voluntary_csw:" << dvctx << ", involuntary_csw:" << divctx;
# else
msgbuf << "Event call took " << (et - st) << "s";
# endif /* RUSAGE_THREAD */
2013-03-16 21:18:53 +01:00
Log(LogWarning, "base", msgbuf.str());
2012-08-03 13:19:55 +02:00
}
#endif /* _DEBUG */
2012-07-16 22:00:50 +02:00
}
2013-03-23 12:23:13 +01:00
UpdateThreadUtilization(tid, ThreadDead);
2013-08-27 15:57:00 +02:00
m_ThreadStats[tid].Zombie = false;
2012-06-24 02:56:48 +02:00
}
2012-09-14 14:41:17 +02:00
/**
2013-03-25 18:36:15 +01:00
* Appends a work item to the work queue. Work items will be processed in FIFO order.
2012-09-14 14:41:17 +02:00
*
2013-03-25 18:36:15 +01:00
* @param callback The callback function for the work item.
* @returns true if the item was queued, false otherwise.
2012-09-14 14:41:17 +02:00
*/
bool ThreadPool::Post(const ThreadPool::WorkFunction& callback)
2012-06-24 02:56:48 +02:00
{
2013-02-17 19:14:34 +01:00
boost::mutex::scoped_lock lock(m_Mutex);
2013-03-23 12:23:13 +01:00
if (m_Stopped)
return false;
2013-03-23 12:23:13 +01:00
2013-03-25 18:36:15 +01:00
WorkItem wi;
wi.Callback = callback;
wi.Timestamp = Utility::GetTime();
2013-03-25 18:36:15 +01:00
m_WorkItems.push_back(wi);
2013-04-19 12:58:16 +02:00
m_WorkCV.notify_one();
return true;
2013-02-26 10:13:54 +01:00
}
2013-02-20 19:52:25 +01:00
2013-03-25 18:36:15 +01:00
void ThreadPool::ManagerThreadProc(void)
2013-02-26 10:13:54 +01:00
{
2013-03-25 18:36:15 +01:00
std::ostringstream idbuf;
idbuf << "TP " << this << " Manager";
Utility::SetThreadName(idbuf.str());
2013-03-06 11:03:50 +01:00
for (;;) {
size_t pending, alive;
2013-03-23 20:38:41 +01:00
double avg_latency, max_latency;
2013-03-25 18:36:15 +01:00
double utilization = 0;
2013-02-26 10:13:54 +01:00
2013-03-06 11:03:50 +01:00
{
boost::mutex::scoped_lock lock(m_Mutex);
2013-04-19 12:58:16 +02:00
m_MgmtCV.timed_wait(lock, boost::posix_time::seconds(5));
if (m_Stopped)
break;
2013-03-25 18:36:15 +01:00
pending = m_WorkItems.size();
2013-03-23 12:23:13 +01:00
alive = 0;
for (size_t i = 0; i < sizeof(m_ThreadStats) / sizeof(m_ThreadStats[0]); i++) {
2013-08-27 15:57:00 +02:00
if (m_ThreadStats[i].State != ThreadDead && !m_ThreadStats[i].Zombie) {
2013-03-23 12:23:13 +01:00
alive++;
utilization += m_ThreadStats[i].Utilization * 100;
}
}
2013-03-25 18:36:15 +01:00
utilization /= alive;
if (m_TaskCount > 0)
avg_latency = m_WaitTime / (m_TaskCount * 1.0);
2013-03-23 12:23:13 +01:00
else
avg_latency = 0;
if (utilization < 60 || utilization > 80 || alive < 2) {
2013-08-27 15:57:00 +02:00
double wthreads = ceil((utilization * alive) / 80.0);
if (!finite(wthreads))
wthreads = 0;
int tthreads = wthreads - alive;
2013-03-23 20:38:41 +01:00
2013-03-25 18:36:15 +01:00
/* Don't ever kill the last 2 threads. */
if (alive + tthreads < 2)
tthreads = 2 - alive;
2013-03-25 18:36:15 +01:00
/* Spawn more workers if there are outstanding work items. */
if (tthreads > 0 && pending > 0)
tthreads = 8;
2013-08-27 15:57:00 +02:00
std::ostringstream msgbuf;
msgbuf << "Thread pool; current: " << alive << "; adjustment: " << tthreads;
Log(LogDebug, "base", msgbuf.str());
for (int i = 0; i < -tthreads; i++)
KillWorker();
for (int i = 0; i < tthreads; i++)
2013-03-23 12:23:13 +01:00
SpawnWorker();
}
m_WaitTime = 0;
m_ServiceTime = 0;
m_TaskCount = 0;
max_latency = m_MaxLatency;
m_MaxLatency = 0;
2013-03-23 12:23:13 +01:00
}
2013-03-23 12:23:13 +01:00
std::ostringstream msgbuf;
2013-03-25 18:36:15 +01:00
msgbuf << "Pending tasks: " << pending << "; Average latency: "
<< (long)(avg_latency * 1000) << "ms"
<< "; Max latency: " << (long)(max_latency * 1000) << "ms"
<< "; Threads: " << alive
<< "; Pool utilization: " << utilization << "%";
2013-03-23 12:23:13 +01:00
Log(LogInformation, "base", msgbuf.str());
}
}
2013-03-23 12:23:13 +01:00
/**
* Note: Caller must hold m_Mutex
*/
2013-03-25 18:36:15 +01:00
void ThreadPool::SpawnWorker(void)
2013-03-23 12:23:13 +01:00
{
for (size_t i = 0; i < sizeof(m_ThreadStats) / sizeof(m_ThreadStats[0]); i++) {
if (m_ThreadStats[i].State == ThreadDead) {
2013-03-23 20:38:41 +01:00
Log(LogDebug, "debug", "Spawning worker thread.");
m_ThreadStats[i] = ThreadStats(ThreadIdle);
2013-03-25 18:36:15 +01:00
boost::thread worker(boost::bind(&ThreadPool::QueueThreadProc, this, i));
2013-03-23 12:23:13 +01:00
worker.detach();
2013-03-23 12:23:13 +01:00
break;
2013-03-06 11:03:50 +01:00
}
}
2012-06-24 02:56:48 +02:00
}
2013-03-23 12:23:13 +01:00
/**
* Note: Caller must hold m_Mutex.
*/
2013-03-25 18:36:15 +01:00
void ThreadPool::KillWorker(void)
2013-03-23 12:23:13 +01:00
{
2013-08-27 15:57:00 +02:00
for (size_t i = 0; i < sizeof(m_ThreadStats) / sizeof(m_ThreadStats[0]); i++) {
if (m_ThreadStats[i].State == ThreadIdle && !m_ThreadStats[i].Zombie) {
Log(LogDebug, "base", "Killing worker thread.");
2013-03-23 12:23:13 +01:00
2013-08-27 15:57:00 +02:00
m_ThreadStats[i].Zombie = true;
m_WorkCV.notify_all();
break;
}
}
2013-03-23 12:23:13 +01:00
}
void ThreadPool::StatsThreadProc(void)
{
std::ostringstream idbuf;
idbuf << "TP " << this << " Stats";
Utility::SetThreadName(idbuf.str());
for (;;) {
2013-04-19 12:58:16 +02:00
boost::mutex::scoped_lock lock(m_Mutex);
2013-04-19 12:58:16 +02:00
m_MgmtCV.timed_wait(lock, boost::posix_time::milliseconds(250));
2013-04-19 12:58:16 +02:00
if (m_Stopped)
break;
for (size_t i = 0; i < sizeof(m_ThreadStats) / sizeof(m_ThreadStats[0]); i++)
2013-04-19 12:58:16 +02:00
UpdateThreadUtilization(i);
}
}
/**
* Note: Caller must hold m_Mutex.
*/
void ThreadPool::UpdateThreadUtilization(int tid, ThreadState state)
{
double utilization;
switch (m_ThreadStats[tid].State) {
case ThreadDead:
return;
case ThreadIdle:
utilization = 0;
break;
case ThreadBusy:
utilization = 1;
break;
default:
ASSERT(0);
}
double now = Utility::GetTime();
double time = now - m_ThreadStats[tid].LastUpdate;
const double avg_time = 5.0;
if (time > avg_time)
time = avg_time;
m_ThreadStats[tid].Utilization = (m_ThreadStats[tid].Utilization * (avg_time - time) + utilization * time) / avg_time;
m_ThreadStats[tid].LastUpdate = now;
if (state != ThreadUnspecified)
m_ThreadStats[tid].State = state;
}