Various bugfixes.

components/notification/notificationcomponent.cpp

@@ -60,6 +60,9 @@ void NotificationComponent::NotificationTimerHandler(void)
 		Service::Ptr service = dynamic_pointer_cast<Service>(object);
 		bool reachable = service->IsReachable();
 
+		bool send_notification;
+
+		{
 			ObjectLock olock(service);
 
 			if (service->GetStateType() == StateTypeSoft)
@@ -74,7 +77,10 @@ void NotificationComponent::NotificationTimerHandler(void)
 			if (service->GetLastNotification() > now - service->GetNotificationInterval())
 				continue;
 
-		if (reachable && !service->IsInDowntime() && !service->IsAcknowledged())
+			send_notification = reachable && !service->IsInDowntime() && !service->IsAcknowledged();
+		}
+
+		if (send_notification)
 			service->RequestNotifications(NotificationProblem);
 	}
 }
@@ -99,5 +105,8 @@ void NotificationComponent::SendNotificationsRequestHandler(const Endpoint::Ptr&
 
 	Service::Ptr service = Service::GetByName(svc);
 
+	if (!service)
+		return;
+
 	service->SendNotifications(static_cast<NotificationType>(type));
 }

configure.ac

@@ -75,8 +75,8 @@ AC_CHECK_FUNCS([backtrace_symbols execvpe pipe2])
 AC_MSG_CHECKING(whether to enable debugging)
 AC_ARG_ENABLE(debug, [  --enable-debug=[no/yes]   turn on debugging (default=no)],, enable_debug=no)
 if test "x$enable_debug" = "xyes"; then
-	CFLAGS="$CFLAGS -g -O0 -D_DEBUG"
+	CFLAGS="$CFLAGS -g -O0 -D_DEBUG -Wall -Wextra"
-	CXXFLAGS="$CXXFLAGS -g -O0 -D_DEBUG"
+	CXXFLAGS="$CXXFLAGS -g -O0 -D_DEBUG -Wall -Wextra"
 else
 	CFLAGS="$CFLAGS -DNDEBUG"
 	CXXFLAGS="$CXXFLAGS -DNDEBUG"

lib/base/application.cpp

@@ -109,6 +109,7 @@ void Application::ShutdownTimerHandler(void)
 	if (m_ShuttingDown) {
 		Logger::Write(LogInformation, "base", "Shutting down Icinga...");
 		Application::GetInstance()->OnShutdown();
+
 		DynamicObject::DeactivateObjects();
 		GetEQ().Stop();
 		m_ShuttingDown = false;
@@ -285,17 +286,17 @@ void Application::SigIntHandler(int signum)
 {
 	assert(signum == SIGINT);
 
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = SIG_DFL;
+	sigaction(SIGINT, &sa, NULL);
+
 	Application::Ptr instance = Application::GetInstance();
 
 	if (!instance)
 		return;
 
 	instance->RequestShutdown();
-
-	struct sigaction sa;
-	memset(&sa, 0, sizeof(sa));
-	sa.sa_handler = SIG_DFL;
-	sigaction(SIGINT, &sa, NULL);
 }
 
 /**
@@ -307,6 +308,11 @@ void Application::SigAbrtHandler(int signum)
 {
 	assert(signum == SIGABRT);
 
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = SIG_DFL;
+	sigaction(SIGABRT, &sa, NULL);
+
 	std::cerr << "Caught SIGABRT." << std::endl;
 
 	Utility::PrintStacktrace(std::cerr, 1);
@@ -337,6 +343,13 @@ BOOL WINAPI Application::CtrlHandler(DWORD type)
  */
 void Application::ExceptionHandler(void)
 {
+#ifndef _WIN32
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = SIG_DFL;
+	sigaction(SIGABRT, &sa, NULL);
+#endif /* _WIN32 */
+
 	try {
 		throw;
 	} catch (const std::exception& ex) {
@@ -349,13 +362,6 @@ void Application::ExceptionHandler(void)
 
 	DisplayBugMessage();
 
-#ifndef _WIN32
-	struct sigaction sa;
-	memset(&sa, 0, sizeof(sa));
-	sa.sa_handler = SIG_DFL;
-	sigaction(SIGABRT, &sa, NULL);
-#endif /* _WIN32 */
-
 	abort();
 }
 

lib/base/dynamicobject.cpp

@@ -128,7 +128,7 @@ void DynamicObject::ApplyUpdate(const Dictionary::Ptr& serializedUpdate,
 	boost::mutex::scoped_lock lock(m_AttributeMutex);
 
 	if ((allowedTypes & Attribute_Config) != 0 && !configTxValue.IsEmpty()) {
-		double oldConfigTx, configTx = configTxValue;
+		double configTx = configTxValue;
 
 		if (configTx > m_ConfigTx) {
 			DynamicObject::AttributeIterator at;
@@ -226,7 +226,7 @@ void DynamicObject::Set(const String& name, const Value& data)
  */
 void DynamicObject::Touch(const String& name)
 {
-	assert(!OwnsLock());
+	assert(OwnsLock());
 
 	boost::mutex::scoped_lock lock(m_AttributeMutex);
 
@@ -284,7 +284,7 @@ void DynamicObject::InternalSetAttribute(const String& name, const Value& data,
 			m_ConfigTx = tx;
 	}
 
-	if (m_Registered) {
+	if (IsRegistered()) {
 		/* We can't call GetSelf() in the constructor or destructor.
 		 * The Register() function will take care of adding this
 		 * object to the list of modified objects later on if we can't
@@ -352,6 +352,7 @@ bool DynamicObject::IsAbstract(void) const
  */
 bool DynamicObject::IsRegistered(void) const
 {
+	ObjectLock olock(GetType());
 	return m_Registered;
 }
 
@@ -392,11 +393,6 @@ void DynamicObject::OnRegistrationCompleted(void)
 {
 	assert(!OwnsLock());
 
-	{
-		ObjectLock olock(this);
-		m_Registered = true;
-	}
-
 	Start();
 
 	OnRegistered(GetSelf());
@@ -406,11 +402,6 @@ void DynamicObject::OnUnregistrationCompleted(void)
 {
 	assert(!OwnsLock());
 
-	{
-		ObjectLock olock(this);
-		m_Registered = false;
-	}
-
 	OnUnregistered(GetSelf());
 }
 

lib/base/dynamicobject.h

@@ -117,7 +117,7 @@ private:
 	Attribute<String> m_Source;
 	Attribute<Dictionary::Ptr> m_Methods;
 
-	bool m_Registered;
+	bool m_Registered;	/**< protected by the type mutex */
 
 	static double m_CurrentTx;
 

lib/base/dynamictype.cpp

@@ -83,6 +83,7 @@ void DynamicType::RegisterObject(const DynamicObject::Ptr& object)
 {
 	String name = object->GetName();
 
+	{
 		ObjectLock olock(this);
 
 		ObjectMap::iterator it = m_ObjectMap.find(name);
@@ -97,16 +98,23 @@ void DynamicType::RegisterObject(const DynamicObject::Ptr& object)
 		m_ObjectMap[name] = object;
 		m_ObjectSet.insert(object);
 
+		object->m_Registered = true;
+	}
+
 	object->OnRegistrationCompleted();
 }
 
 void DynamicType::UnregisterObject(const DynamicObject::Ptr& object)
 {
+	{
 		ObjectLock olock(this);
 
 		m_ObjectMap.erase(object->GetName());
 		m_ObjectSet.erase(object);
 
+		object->m_Registered = false;
+	}
+
 	object->OnUnregistrationCompleted();
 }
 

lib/base/eventqueue.cpp

@@ -25,20 +25,20 @@ using namespace icinga;
  * @threadsafety Always.
  */
 EventQueue::EventQueue(void)
-	: m_Stopped(false), m_LastReport(0)
+	: m_Stopped(false)
 {
-	int thread_count = thread::hardware_concurrency();
+	unsigned int threads = thread::hardware_concurrency();
 
-	if (thread_count < 4)
+	if (threads == 0)
-		thread_count = 4;
+		threads = 1;
 
-	for (int i = 0; i < thread_count; i++)
+	threads *= 8;
+
+	for (unsigned int i = 0; i < threads; i++)
 		m_Threads.create_thread(boost::bind(&EventQueue::QueueThreadProc, this));
 
-	m_ReportTimer = boost::make_shared<Timer>();
+	thread reportThread(boost::bind(&EventQueue::ReportThreadProc, this));
-	m_ReportTimer->OnTimerExpired.connect(boost::bind(&EventQueue::ReportTimerHandler, this));
+	reportThread.detach();
-	m_ReportTimer->SetInterval(5);
-	m_ReportTimer->Start();
 }
 
 /**
@@ -46,8 +46,6 @@ EventQueue::EventQueue(void)
  */
 EventQueue::~EventQueue(void)
 {
-	m_ReportTimer->Stop();
-
 	Stop();
 	Join();
 }
@@ -95,17 +93,39 @@ void EventQueue::QueueThreadProc(void)
 		}
 
 		BOOST_FOREACH(const Callback& ev, events) {
+#ifdef _DEBUG
+			struct rusage usage_start, usage_end;
+
 			double st = Utility::GetTime();
+			(void) getrusage(RUSAGE_THREAD, &usage_start);
+#endif /* _DEBUG */
 
 			ev();
 
+#ifdef _DEBUG
 			double et = Utility::GetTime();
+			(void) getrusage(RUSAGE_THREAD, &usage_end);
 
-			if (et - st > 0.25) {
+			double duser = (usage_end.ru_utime.tv_sec - usage_start.ru_utime.tv_sec) +
+			    (usage_end.ru_utime.tv_usec - usage_start.ru_utime.tv_usec) / 1000000.0;
+
+			double dsys = (usage_end.ru_stime.tv_sec - usage_start.ru_stime.tv_sec) +
+			    (usage_end.ru_stime.tv_usec - usage_start.ru_stime.tv_usec) / 1000000.0;
+
+			double dwait = (et - st) - (duser + dsys);
+
+			int dminfaults = usage_end.ru_minflt - usage_start.ru_minflt;
+			int dmajfaults = usage_end.ru_majflt - usage_start.ru_majflt;
+
+			int dvctx = usage_end.ru_nvcsw - usage_start.ru_nvcsw;
+			int divctx = usage_end.ru_nivcsw - usage_start.ru_nivcsw;
+
+			if (et - st > 0.5) {
 				stringstream msgbuf;
-				msgbuf << "Event call took " << et - st << " seconds.";
+				msgbuf << "Event call took user:" << duser << "s, system:" << dsys << "s, wait:" << dwait << "s, minor_faults:" << dminfaults << ", major_faults:" << dmajfaults << ", voluntary_csw:" << dvctx << ", involuntary_csw:" << divctx;
 				Logger::Write(LogWarning, "base", msgbuf.str());
 			}
+#endif /* _DEBUG */
 		}
 	}
 }
@@ -123,8 +143,11 @@ void EventQueue::Post(const EventQueue::Callback& callback)
 	m_CV.notify_one();
 }
 
-void EventQueue::ReportTimerHandler(void)
+void EventQueue::ReportThreadProc(void)
 {
+	for (;;) {
+		Utility::Sleep(5);
+
 		int pending;
 
 		{
@@ -132,6 +155,6 @@ void EventQueue::ReportTimerHandler(void)
 			pending = m_Events.size();
 		}
 
-	if (pending > 1000)
+		Logger::Write(LogInformation, "base", "Pending tasks: " + Convert::ToString(pending));
-		Logger::Write(LogCritical, "base", "More than 1000 pending events: " + Convert::ToString(pending));
+	}
 }

lib/base/eventqueue.h

@@ -49,14 +49,11 @@ private:
 	boost::mutex m_Mutex;
 	condition_variable m_CV;
 
-	double m_LastReport;
-	shared_ptr<Timer> m_ReportTimer;
-
 	bool m_Stopped;
 	vector<Callback> m_Events;
 
 	void QueueThreadProc(void);
-	void ReportTimerHandler(void);
+	void ReportThreadProc(void);
 };
 
 }

lib/base/object.h

@@ -117,7 +117,7 @@ private:
 
 	mutable MutexType m_Mutex;
 
-	friend class ObjectLock;
+	friend struct ObjectLock;
 };
 
 /**

lib/base/objectlock.cpp

@@ -56,7 +56,6 @@ void ObjectLock::Lock(void)
 		boost::mutex::scoped_lock lock(Object::m_DebugMutex);
 		m_Object->m_Locked = true;
 		m_Object->m_LockOwner = boost::this_thread::get_id();
-		m_TS = Utility::GetTime();
 	}
 #endif /* _DEBUG */
 }
@@ -67,17 +66,9 @@ void ObjectLock::Unlock(void)
 	{
 		boost::mutex::scoped_lock lock(Object::m_DebugMutex);
 
-		if (m_Lock.owns_lock()) {
+		if (m_Lock.owns_lock())
-			double dt = Utility::GetTime() - m_TS;
-
-			if (dt > 0.05) {
-				std::cerr << "Held object lock for " << dt << " seconds at:";
-				Utility::PrintStacktrace(std::cerr);
-			}
-
 			m_Object->m_Locked = false;
 	}
-	}
 #endif /* _DEBUG */
 
 	m_Lock = Object::MutexType::scoped_lock();

lib/base/objectlock.h

@@ -39,10 +39,6 @@ public:
 private:
 	const Object *m_Object;
 	Object::MutexType::scoped_lock m_Lock;
-
-#ifdef _DEBUG
-	double m_TS;
-#endif /* _DEBUG */
 };
 
 }

lib/base/process-unix.cpp

@@ -24,6 +24,7 @@
 
 using namespace icinga;
 
+condition_variable Process::m_CV;
 int Process::m_TaskFd;
 Timer::Ptr Process::m_StatusTimer;
 extern char **environ;
@@ -47,7 +48,12 @@ void Process::Initialize(void)
 
 	m_TaskFd = fds[1];
 
-	for (int i = 0; i < thread::hardware_concurrency(); i++) {
+	unsigned int threads = thread::hardware_concurrency();
+
+	if (threads == 0)
+		threads = 2;
+
+	for (unsigned int i = 0; i < threads; i++) {
 		int childTaskFd = dup(fds[0]);
 
 		if (childTaskFd < 0)
@@ -104,34 +110,50 @@ void Process::WorkerThreadProc(int taskFd)
 		if (rc == 0)
 			continue;
 
-
 		for (int i = 0; i < idx; i++) {
 			if ((pfds[i].revents & (POLLIN|POLLHUP)) == 0)
 				continue;
 
-			while (pfds[i].fd == taskFd && tasks.size() < MaxTasksPerThread) {
+			if (pfds[i].fd == taskFd) {
-				Process::Ptr task;
+				vector<Process::Ptr> new_tasks;
 
-				{
+				int want = MaxTasksPerThread - tasks.size();
+
+				if (want > m_Tasks.size())
+					want = m_Tasks.size();
+
+				if (want > 0) {
 					boost::mutex::scoped_lock lock(m_Mutex);
 
 					/* Read one byte for every task we take from the pending tasks list. */
-					char buffer;
+					char buffer[MaxTasksPerThread];
-					int rc = read(taskFd, &buffer, sizeof(buffer));
 
-					if (rc < 0) {
+					assert(want =< sizeof(buffer));
+
+					int have = read(taskFd, &buffer, want);
+
+					if (have < 0) {
 						if (errno == EAGAIN)
 							break; /* Someone else was faster and took our task. */
 
 						BOOST_THROW_EXCEPTION(PosixException("read() failed.", errno));
 					}
 
+					while (have > 0) {
 						assert(!m_Tasks.empty());
 
-					task = m_Tasks.front();
+						Process::Ptr task = m_Tasks.front();
 						m_Tasks.pop_front();
+
+						new_tasks.push_back(task);
+
+						have--;
 					}
 
+					m_CV.notify_all();
+				}
+
+				BOOST_FOREACH(const Process::Ptr& task, new_tasks) {
 					try {
 						task->InitTask();
 
@@ -144,6 +166,9 @@ void Process::WorkerThreadProc(int taskFd)
 					}
 				}
 
+				continue;
+			}
+
 			it = tasks.find(pfds[i].fd);
 
 			if (it == tasks.end())
@@ -161,14 +186,23 @@ void Process::WorkerThreadProc(int taskFd)
 	}
 }
 
-void Process::NotifyWorker(void)
+void Process::QueueTask(void)
 {
+	{
+		boost::mutex::scoped_lock lock(m_Mutex);
+
+		while (m_Tasks.size() >= PIPE_BUF)
+			m_CV.wait(lock);
+
+		m_Tasks.push_back(GetSelf());
+
 		/**
 		 * This little gem which is commonly known as the "self-pipe trick"
 		 * takes care of waking up the select() call in the worker thread.
 		 */
 		if (write(m_TaskFd, "T", 1) < 0)
 			BOOST_THROW_EXCEPTION(PosixException("write() failed.", errno));
+	}
 }
 
 void Process::InitTask(void)
@@ -196,7 +230,7 @@ void Process::InitTask(void)
 	// build argv
 	char **argv = new char *[m_Arguments.size() + 1];
 
-	for (int i = 0; i < m_Arguments.size(); i++)
+	for (unsigned int i = 0; i < m_Arguments.size(); i++)
 		argv[i] = strdup(m_Arguments[i].CStr());
 
 	argv[m_Arguments.size()] = NULL;

lib/base/process-windows.cpp

@@ -32,7 +32,7 @@ void Process::WorkerThreadProc(void)
 	// TODO: implement
 }
 
-void Process::NotifyWorker(void)
+void Process::QueueTask(void)
 {
 	// TODO: implement
 }

lib/base/process.cpp

@@ -67,10 +67,5 @@ vector<String> Process::SplitCommand(const Value& command)
 
 void Process::Run(void)
 {
-	{
+	QueueTask();
-		boost::mutex::scoped_lock lock(m_Mutex);
-		m_Tasks.push_back(GetSelf());
-	}
-
-	NotifyWorker();
 }

lib/base/process.h

@@ -48,7 +48,7 @@ public:
 	typedef shared_ptr<Process> Ptr;
 	typedef weak_ptr<Process> WeakPtr;
 
-	static const deque<Process::Ptr>::size_type MaxTasksPerThread = 128;
+	static const deque<Process::Ptr>::size_type MaxTasksPerThread = 512;
 
 	Process(const vector<String>& arguments, const Dictionary::Ptr& extraEnvironment = Dictionary::Ptr());
 
@@ -71,12 +71,13 @@ private:
 	static boost::mutex m_Mutex;
 	static deque<Process::Ptr> m_Tasks;
 #ifndef _WIN32
+	static condition_variable m_CV;
 	static int m_TaskFd;
 
 	static Timer::Ptr m_StatusTimer;
 #endif /* _WIN32 */
 
-	static void NotifyWorker(void);
+	void QueueTask(void);
 
 	void SpawnTask(void);
 

lib/base/ringbuffer.cpp

@@ -42,10 +42,10 @@ void RingBuffer::InsertValue(RingBuffer::SizeType tv, int num)
 {
 	ObjectLock olock(this);
 
-	vector<int>::size_type offsetTarget = tv % m_Slots.size();
+	RingBuffer::SizeType offsetTarget = tv % m_Slots.size();
 
 	if (tv > m_TimeValue) {
-		vector<int>::size_type offset = m_TimeValue % m_Slots.size();
+		RingBuffer::SizeType offset = m_TimeValue % m_Slots.size();
 
 		/* walk towards the target offset, resetting slots to 0 */
 		while (offset != offsetTarget) {

lib/base/ringbuffer.h

@@ -44,7 +44,7 @@ public:
 
 private:
 	vector<int> m_Slots;
-	int m_TimeValue;
+	SizeType m_TimeValue;
 };
 
 }

lib/base/scriptfunction.cpp

@@ -72,8 +72,6 @@ ScriptFunction::Ptr ScriptFunction::GetByName(const String& name)
  */
 void ScriptFunction::Invoke(const ScriptTask::Ptr& task, const vector<Value>& arguments)
 {
-	ObjectLock olock(this);
-
 	m_Callback(task, arguments);
 }
 

lib/base/timer.cpp

@@ -94,8 +94,7 @@ void Timer::Call(void)
 
 	OnTimerExpired(self);
 
-	/* Re-enable the timer so it can be called again. */
+	Reschedule();
-	Start();
 }
 
 /**
@@ -285,7 +284,6 @@ void Timer::TimerThreadProc(void)
 
 		/* Remove the timer from the list so it doesn't get called again
 		 * until the current call is completed. */
-		timer->m_Started = false;
 		m_Timers.erase(timer);
 
 		lock.unlock();

lib/icinga/externalcommandprocessor.cpp

@@ -87,7 +87,7 @@ void ExternalCommandProcessor::Execute(double time, const String& command, const
  */
 void ExternalCommandProcessor::Initialize(void)
 {
-	RegisterCommand("PROCESS_HOST_CHECK_RESULT", &ExternalCommandProcessor::ProcessServiceCheckResult);
+	RegisterCommand("PROCESS_HOST_CHECK_RESULT", &ExternalCommandProcessor::ProcessHostCheckResult);
 	RegisterCommand("PROCESS_SERVICE_CHECK_RESULT", &ExternalCommandProcessor::ProcessServiceCheckResult);
 	RegisterCommand("SCHEDULE_HOST_CHECK", &ExternalCommandProcessor::ScheduleHostCheck);
 	RegisterCommand("SCHEDULE_FORCED_HOST_CHECK", &ExternalCommandProcessor::ScheduleForcedHostCheck);
@@ -167,8 +167,8 @@ void ExternalCommandProcessor::ProcessHostCheckResult(double time, const vector<
 	if (!hc->GetEnablePassiveChecks())
 		BOOST_THROW_EXCEPTION(invalid_argument("Got passive check result for host '" + arguments[0] + "' which has passive checks disabled."));
 
-	int exitStatus = Convert::ToDouble(arguments[2]);
+	int exitStatus = Convert::ToDouble(arguments[1]);
-	Dictionary::Ptr result = PluginCheckTask::ParseCheckOutput(arguments[3]);
+	Dictionary::Ptr result = PluginCheckTask::ParseCheckOutput(arguments[2]);
 	result->Set("state", PluginCheckTask::ExitStatusToState(exitStatus));
 
 	result->Set("schedule_start", time);
@@ -180,10 +180,14 @@ void ExternalCommandProcessor::ProcessHostCheckResult(double time, const vector<
 	Logger::Write(LogInformation, "icinga", "Processing passive check result for host '" + arguments[0] + "'");
 	hc->ProcessCheckResult(result);
 
+	{
+		ObjectLock olock(hc);
+
 		/* Reschedule the next check. The side effect of this is that for as long
 		 * as we receive passive results for a service we won't execute any
 		 * active checks. */
 		hc->SetNextCheck(Utility::GetTime() + hc->GetCheckInterval());
+	}
 }
 
 void ExternalCommandProcessor::ProcessServiceCheckResult(double time, const vector<String>& arguments)
@@ -209,10 +213,14 @@ void ExternalCommandProcessor::ProcessServiceCheckResult(double time, const vect
 	Logger::Write(LogInformation, "icinga", "Processing passive check result for service '" + arguments[1] + "'");
 	service->ProcessCheckResult(result);
 
+	{
+		ObjectLock olock(service);
+
 		/* Reschedule the next check. The side effect of this is that for as long
 		 * as we receive passive results for a service we won't execute any
 		 * active checks. */
 		service->SetNextCheck(Utility::GetTime() + service->GetCheckInterval());
+	}
 }
 
 void ExternalCommandProcessor::ScheduleHostCheck(double, const vector<String>& arguments)
@@ -233,7 +241,12 @@ void ExternalCommandProcessor::ScheduleHostCheck(double, const vector<String>& a
 	}
 
 	Logger::Write(LogInformation, "icinga", "Rescheduling next check for host '" + arguments[0] + "'");
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetNextCheck(planned_check);
+	}
 }
 
 void ExternalCommandProcessor::ScheduleForcedHostCheck(double, const vector<String>& arguments)
@@ -246,8 +259,13 @@ void ExternalCommandProcessor::ScheduleForcedHostCheck(double, const vector<Stri
 	Service::Ptr hc = host->GetHostCheckService();
 
 	Logger::Write(LogInformation, "icinga", "Rescheduling next check for host '" + arguments[0] + "'");
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetForceNextCheck(true);
 		hc->SetNextCheck(Convert::ToDouble(arguments[1]));
+	}
 }
 
 void ExternalCommandProcessor::ScheduleSvcCheck(double, const vector<String>& arguments)
@@ -266,7 +284,12 @@ void ExternalCommandProcessor::ScheduleSvcCheck(double, const vector<String>& ar
 	}
 
 	Logger::Write(LogInformation, "icinga", "Rescheduling next check for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetNextCheck(planned_check);
+	}
 }
 
 void ExternalCommandProcessor::ScheduleForcedSvcCheck(double, const vector<String>& arguments)
@@ -277,8 +300,13 @@ void ExternalCommandProcessor::ScheduleForcedSvcCheck(double, const vector<Strin
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Rescheduling next check for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetForceNextCheck(true);
 		service->SetNextCheck(Convert::ToDouble(arguments[2]));
+	}
 }
 
 void ExternalCommandProcessor::EnableHostCheck(double, const vector<String>& arguments)
@@ -291,8 +319,14 @@ void ExternalCommandProcessor::EnableHostCheck(double, const vector<String>& arg
 	Logger::Write(LogInformation, "icinga", "Enabling active checks for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnableActiveChecks(true);
+	}
 }
 
 void ExternalCommandProcessor::DisableHostCheck(double, const vector<String>& arguments)
@@ -305,8 +339,14 @@ void ExternalCommandProcessor::DisableHostCheck(double, const vector<String>& ar
 	Logger::Write(LogInformation, "icinga", "Disabling active checks for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnableActiveChecks(false);
+	}
 }
 
 void ExternalCommandProcessor::EnableSvcCheck(double, const vector<String>& arguments)
@@ -317,7 +357,12 @@ void ExternalCommandProcessor::EnableSvcCheck(double, const vector<String>& argu
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Enabling active checks for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnableActiveChecks(true);
+	}
 }
 
 void ExternalCommandProcessor::DisableSvcCheck(double, const vector<String>& arguments)
@@ -328,7 +373,12 @@ void ExternalCommandProcessor::DisableSvcCheck(double, const vector<String>& arg
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Disabling active checks for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnableActiveChecks(false);
+	}
 }
 
 void ExternalCommandProcessor::ShutdownProcess(double, const vector<String>&)
@@ -348,9 +398,14 @@ void ExternalCommandProcessor::ScheduleForcedHostSvcChecks(double, const vector<
 
 	BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 		Logger::Write(LogInformation, "icinga", "Rescheduling next check for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetNextCheck(planned_check);
 			service->SetForceNextCheck(true);
 		}
+	}
 }
 
 void ExternalCommandProcessor::ScheduleHostSvcChecks(double, const vector<String>& arguments)
@@ -370,8 +425,13 @@ void ExternalCommandProcessor::ScheduleHostSvcChecks(double, const vector<String
 		}
 
 		Logger::Write(LogInformation, "icinga", "Rescheduling next check for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetNextCheck(planned_check);
 		}
+	}
 }
 
 void ExternalCommandProcessor::EnableHostSvcChecks(double, const vector<String>& arguments)
@@ -396,8 +456,13 @@ void ExternalCommandProcessor::DisableHostSvcChecks(double, const vector<String>
 
 	BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 		Logger::Write(LogInformation, "icinga", "Disabling active checks for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetEnableActiveChecks(false);
 		}
+	}
 }
 
 void ExternalCommandProcessor::AcknowledgeSvcProblem(double, const vector<String>& arguments)
@@ -413,7 +478,12 @@ void ExternalCommandProcessor::AcknowledgeSvcProblem(double, const vector<String
 		BOOST_THROW_EXCEPTION(invalid_argument("The service '" + arguments[1] + "' is OK."));
 
 	Logger::Write(LogInformation, "icinga", "Setting acknowledgement for service '" + service->GetName() + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->AcknowledgeProblem(sticky ? AcknowledgementSticky : AcknowledgementNormal);
+	}
 }
 
 void ExternalCommandProcessor::AcknowledgeSvcProblemExpire(double, const vector<String>& arguments)
@@ -430,7 +500,12 @@ void ExternalCommandProcessor::AcknowledgeSvcProblemExpire(double, const vector<
 		BOOST_THROW_EXCEPTION(invalid_argument("The service '" + arguments[1] + "' is OK."));
 
 	Logger::Write(LogInformation, "icinga", "Setting timed acknowledgement for service '" + service->GetName() + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->AcknowledgeProblem(sticky ? AcknowledgementSticky : AcknowledgementNormal, timestamp);
+	}
 }
 
 void ExternalCommandProcessor::RemoveSvcAcknowledgement(double, const vector<String>& arguments)
@@ -441,7 +516,12 @@ void ExternalCommandProcessor::RemoveSvcAcknowledgement(double, const vector<Str
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Removing acknowledgement for service '" + service->GetName() + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->ClearAcknowledgement();
+	}
 }
 
 void ExternalCommandProcessor::AcknowledgeHostProblem(double, const vector<String>& arguments)
@@ -456,6 +536,8 @@ void ExternalCommandProcessor::AcknowledgeHostProblem(double, const vector<Strin
 	Logger::Write(LogInformation, "icinga", "Setting acknowledgement for host '" + host->GetName() + "'");
 	Service::Ptr service = host->GetHostCheckService();
 	if (service) {
+		ObjectLock olock(service);
+
 		if (service->GetState() == StateOK)
 			BOOST_THROW_EXCEPTION(invalid_argument("The host '" + arguments[0] + "' is OK."));
 
@@ -476,6 +558,8 @@ void ExternalCommandProcessor::AcknowledgeHostProblemExpire(double, const vector
 	Logger::Write(LogInformation, "icinga", "Setting timed acknowledgement for host '" + host->GetName() + "'");
 	Service::Ptr service = host->GetHostCheckService();
 	if (service) {
+		ObjectLock olock(service);
+
 		if (service->GetState() == StateOK)
 			BOOST_THROW_EXCEPTION(invalid_argument("The host '" + arguments[0] + "' is OK."));
 
@@ -493,6 +577,8 @@ void ExternalCommandProcessor::RemoveHostAcknowledgement(double, const vector<St
 	Logger::Write(LogInformation, "icinga", "Removing acknowledgement for host '" + host->GetName() + "'");
 	Service::Ptr service = host->GetHostCheckService();
 	if (service) {
+		ObjectLock olock(service);
+
 		service->ClearAcknowledgement();
 	}
 }
@@ -507,9 +593,14 @@ void ExternalCommandProcessor::EnableHostgroupSvcChecks(double, const vector<Str
 	BOOST_FOREACH(const Host::Ptr& host, hg->GetMembers()) {
 		BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 			Logger::Write(LogInformation, "icinga", "Enabling active checks for service '" + service->GetName() + "'");
+
+			{
+				ObjectLock olock(service);
+
 				service->SetEnableActiveChecks(true);
 			}
 		}
+	}
 }
 
 void ExternalCommandProcessor::DisableHostgroupSvcChecks(double, const vector<String>& arguments)
@@ -522,9 +613,14 @@ void ExternalCommandProcessor::DisableHostgroupSvcChecks(double, const vector<St
 	BOOST_FOREACH(const Host::Ptr& host, hg->GetMembers()) {
 		BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 			Logger::Write(LogInformation, "icinga", "Disabling active checks for service '" + service->GetName() + "'");
+
+			{
+				ObjectLock olock(service);
+
 				service->SetEnableActiveChecks(false);
 			}
 		}
+	}
 }
 
 void ExternalCommandProcessor::EnableServicegroupSvcChecks(double, const vector<String>& arguments)
@@ -536,8 +632,13 @@ void ExternalCommandProcessor::EnableServicegroupSvcChecks(double, const vector<
 
 	BOOST_FOREACH(const Service::Ptr& service, sg->GetMembers()) {
 		Logger::Write(LogInformation, "icinga", "Enabling active checks for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetEnableActiveChecks(true);
 		}
+	}
 }
 
 void ExternalCommandProcessor::DisableServicegroupSvcChecks(double, const vector<String>& arguments)
@@ -549,8 +650,13 @@ void ExternalCommandProcessor::DisableServicegroupSvcChecks(double, const vector
 
 	BOOST_FOREACH(const Service::Ptr& service, sg->GetMembers()) {
 		Logger::Write(LogInformation, "icinga", "Disabling active checks for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetEnableActiveChecks(false);
 		}
+	}
 }
 
 void ExternalCommandProcessor::EnablePassiveHostChecks(double, const vector<String>& arguments)
@@ -563,8 +669,14 @@ void ExternalCommandProcessor::EnablePassiveHostChecks(double, const vector<Stri
 	Logger::Write(LogInformation, "icinga", "Enabling passive checks for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnablePassiveChecks(true);
+	}
 }
 
 void ExternalCommandProcessor::DisablePassiveHostChecks(double, const vector<String>& arguments)
@@ -577,8 +689,14 @@ void ExternalCommandProcessor::DisablePassiveHostChecks(double, const vector<Str
 	Logger::Write(LogInformation, "icinga", "Disabling passive checks for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnablePassiveChecks(false);
+	}
 }
 
 void ExternalCommandProcessor::EnablePassiveSvcChecks(double, const vector<String>& arguments)
@@ -589,7 +707,12 @@ void ExternalCommandProcessor::EnablePassiveSvcChecks(double, const vector<Strin
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Enabling passive checks for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnablePassiveChecks(true);
+	}
 }
 
 void ExternalCommandProcessor::DisablePassiveSvcChecks(double, const vector<String>& arguments)
@@ -600,7 +723,12 @@ void ExternalCommandProcessor::DisablePassiveSvcChecks(double, const vector<Stri
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Disabling passive checks for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnablePassiveChecks(false);
+	}
 }
 
 void ExternalCommandProcessor::EnableServicegroupPassiveSvcChecks(double, const vector<String>& arguments)
@@ -612,8 +740,13 @@ void ExternalCommandProcessor::EnableServicegroupPassiveSvcChecks(double, const
 
 	BOOST_FOREACH(const Service::Ptr& service, sg->GetMembers()) {
 		Logger::Write(LogInformation, "icinga", "Enabling passive checks for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetEnablePassiveChecks(true);
 		}
+	}
 }
 
 void ExternalCommandProcessor::DisableServicegroupPassiveSvcChecks(double, const vector<String>& arguments)
@@ -625,8 +758,13 @@ void ExternalCommandProcessor::DisableServicegroupPassiveSvcChecks(double, const
 
 	BOOST_FOREACH(const Service::Ptr& service, sg->GetMembers()) {
 		Logger::Write(LogInformation, "icinga", "Disabling passive checks for service '" + service->GetName() + "'");
+
+		{
+			ObjectLock olock(service);
+
 			service->SetEnablePassiveChecks(true);
 		}
+	}
 }
 
 void ExternalCommandProcessor::EnableHostgroupPassiveSvcChecks(double, const vector<String>& arguments)
@@ -639,9 +777,14 @@ void ExternalCommandProcessor::EnableHostgroupPassiveSvcChecks(double, const vec
 	BOOST_FOREACH(const Host::Ptr& host, hg->GetMembers()) {
 		BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 			Logger::Write(LogInformation, "icinga", "Enabling passive checks for service '" + service->GetName() + "'");
+
+			{
+				ObjectLock olock(service);
+
 				service->SetEnablePassiveChecks(true);
 			}
 		}
+	}
 }
 
 void ExternalCommandProcessor::DisableHostgroupPassiveSvcChecks(double, const vector<String>& arguments)
@@ -654,9 +797,14 @@ void ExternalCommandProcessor::DisableHostgroupPassiveSvcChecks(double, const ve
 	BOOST_FOREACH(const Host::Ptr& host, hg->GetMembers()) {
 		BOOST_FOREACH(const Service::Ptr& service, host->GetServices()) {
 			Logger::Write(LogInformation, "icinga", "Disabling passive checks for service '" + service->GetName() + "'");
+
+			{
+				ObjectLock olock(service);
+
 				service->SetEnablePassiveChecks(false);
 			}
 		}
+	}
 }
 
 void ExternalCommandProcessor::ProcessFile(double, const vector<String>& arguments)
@@ -962,9 +1110,8 @@ void ExternalCommandProcessor::SendCustomHostNotification(double time, const vec
 
 	Logger::Write(LogInformation, "icinga", "Sending custom notification for host " + host->GetName());
 	Service::Ptr service = host->GetHostCheckService();
-	if (service) {
+	if (service)
 		service->RequestNotifications(NotificationCustom);
-	}
 }
 
 void ExternalCommandProcessor::SendCustomSvcNotification(double time, const vector<String>& arguments)
@@ -986,9 +1133,14 @@ void ExternalCommandProcessor::DelayHostNotification(double time, const vector<S
 	Host::Ptr host = Host::GetByName(arguments[0]);
 
 	Logger::Write(LogInformation, "icinga", "Delaying notifications for host " + host->GetName());
-	Service::Ptr service = host->GetHostCheckService();
+	Service::Ptr hc = host->GetHostCheckService();
-	if (service) {
+	if (!hc)
-		service->SetLastNotification(Convert::ToDouble(arguments[1]));
+		return;
+
+	{
+		ObjectLock olock(hc);
+
+		hc->SetLastNotification(Convert::ToDouble(arguments[1]));
 	}
 }
 
@@ -1000,7 +1152,12 @@ void ExternalCommandProcessor::DelaySvcNotification(double time, const vector<St
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Delaying notifications for service " + service->GetName());
+
+	{
+		ObjectLock olock(service);
+
 		service->SetLastNotification(Convert::ToDouble(arguments[2]));
+	}
 }
 
 void ExternalCommandProcessor::EnableHostNotifications(double, const vector<String>& arguments)
@@ -1013,8 +1170,14 @@ void ExternalCommandProcessor::EnableHostNotifications(double, const vector<Stri
 	Logger::Write(LogInformation, "icinga", "Enabling notifications for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnableNotifications(true);
+	}
 }
 
 void ExternalCommandProcessor::DisableHostNotifications(double, const vector<String>& arguments)
@@ -1027,8 +1190,14 @@ void ExternalCommandProcessor::DisableHostNotifications(double, const vector<Str
 	Logger::Write(LogInformation, "icinga", "Disabling notifications for host '" + arguments[0] + "'");
 	Service::Ptr hc = host->GetHostCheckService();
 
-	if (hc)
+	if (!hc)
+		return;
+
+	{
+		ObjectLock olock(hc);
+
 		hc->SetEnableNotifications(false);
+	}
 }
 
 void ExternalCommandProcessor::EnableSvcNotifications(double, const vector<String>& arguments)
@@ -1039,7 +1208,12 @@ void ExternalCommandProcessor::EnableSvcNotifications(double, const vector<Strin
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Enabling notifications for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnableNotifications(true);
+	}
 }
 
 void ExternalCommandProcessor::DisableSvcNotifications(double, const vector<String>& arguments)
@@ -1050,5 +1224,10 @@ void ExternalCommandProcessor::DisableSvcNotifications(double, const vector<Stri
 	Service::Ptr service = Service::GetByNamePair(arguments[0], arguments[1]);
 
 	Logger::Write(LogInformation, "icinga", "Disabling notifications for service '" + arguments[1] + "'");
+
+	{
+		ObjectLock olock(service);
+
 		service->SetEnableNotifications(false);
+	}
 }

lib/icinga/host.cpp

@@ -23,7 +23,8 @@ using namespace icinga;
 
 boost::mutex Host::m_ServiceMutex;
 map<String, map<String, Service::WeakPtr> > Host::m_ServicesCache;
-bool Host::m_ServicesCacheValid = true;
+bool Host::m_ServicesCacheNeedsUpdate = false;
+Timer::Ptr Host::m_ServicesCacheTimer;
 
 REGISTER_SCRIPTFUNCTION("ValidateServiceDictionary", &Host::ValidateServiceDictionary);
 
@@ -134,7 +135,7 @@ bool Host::IsReachable(void) const
 	set<Host::Ptr> parentHosts = GetParentHosts();
 
 	BOOST_FOREACH(const Host::Ptr& host, parentHosts) {
-		Service::Ptr hc = GetHostCheckService();
+		Service::Ptr hc = host->GetHostCheckService();
 
 		/* ignore hosts that don't have a hostcheck */
 		if (!hc)
@@ -325,10 +326,17 @@ void Host::InvalidateServicesCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_ServiceMutex);
 
-		if (m_ServicesCacheValid)
+		if (m_ServicesCacheNeedsUpdate)
-			Utility::QueueAsyncCallback(boost::bind(&Host::RefreshServicesCache));
+			return; /* Someone else has already requested a refresh. */
 
-		m_ServicesCacheValid = false;
+		if (!m_ServicesCacheTimer) {
+			m_ServicesCacheTimer = boost::make_shared<Timer>();
+			m_ServicesCacheTimer->SetInterval(0.5);
+			m_ServicesCacheTimer->OnTimerExpired.connect(boost::bind(&Host::RefreshServicesCache));
+		}
+
+		m_ServicesCacheTimer->Start();
+		m_ServicesCacheNeedsUpdate = true;
 	}
 }
 
@@ -337,12 +345,13 @@ void Host::RefreshServicesCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_ServiceMutex);
 
-		if (m_ServicesCacheValid)
+		assert(m_ServicesCacheNeedsUpdate);
-			return;
+		m_ServicesCacheTimer->Stop();
-
+		m_ServicesCacheNeedsUpdate = false;
-		m_ServicesCacheValid = true;
 	}
 
+	Logger::Write(LogInformation, "icinga", "Updating services cache.");
+
 	map<String, map<String, Service::WeakPtr> > newServicesCache;
 
 	BOOST_FOREACH(const DynamicObject::Ptr& object, DynamicType::GetObjects("Service")) {

lib/icinga/host.h

@@ -80,7 +80,8 @@ private:
 
 	static boost::mutex m_ServiceMutex;
 	static map<String, map<String, weak_ptr<Service> > > m_ServicesCache;
-	static bool m_ServicesCacheValid;
+	static bool m_ServicesCacheNeedsUpdate;
+	static Timer::Ptr m_ServicesCacheTimer;
 
 	void UpdateSlaveServices(void);
 

lib/icinga/hostgroup.cpp

@@ -23,7 +23,8 @@ using namespace icinga;
 
 boost::mutex HostGroup::m_Mutex;
 map<String, vector<Host::WeakPtr> > HostGroup::m_MembersCache;
-bool HostGroup::m_MembersCacheValid = true;
+bool HostGroup::m_MembersCacheNeedsUpdate = false;
+Timer::Ptr HostGroup::m_MembersCacheTimer;
 
 REGISTER_TYPE(HostGroup);
 
@@ -120,10 +121,17 @@ void HostGroup::InvalidateMembersCache(void)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);
 
-	if (m_MembersCacheValid)
+	if (m_MembersCacheNeedsUpdate)
-		Utility::QueueAsyncCallback(boost::bind(&HostGroup::RefreshMembersCache));
+		return; /* Someone else has already requested a refresh. */
 
-	m_MembersCacheValid = false;
+	if (!m_MembersCacheTimer) {
+		m_MembersCacheTimer = boost::make_shared<Timer>();
+		m_MembersCacheTimer->SetInterval(0.5);
+		m_MembersCacheTimer->OnTimerExpired.connect(boost::bind(&HostGroup::RefreshMembersCache));
+	}
+
+	m_MembersCacheTimer->Start();
+	m_MembersCacheNeedsUpdate = true;
 }
 
 /**
@@ -134,10 +142,9 @@ void HostGroup::RefreshMembersCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_Mutex);
 
-		if (m_MembersCacheValid)
+		assert(m_MembersCacheNeedsUpdate);
-			return;
+		m_MembersCacheTimer->Stop();
-
+		m_MembersCacheNeedsUpdate = false;
-		m_MembersCacheValid = true;
 	}
 
 	map<String, vector<Host::WeakPtr> > newMembersCache;

lib/icinga/hostgroup.h

@@ -57,7 +57,8 @@ private:
 
 	static boost::mutex m_Mutex;
 	static map<String, vector<Host::WeakPtr> > m_MembersCache;
-	static bool m_MembersCacheValid;
+	static bool m_MembersCacheNeedsUpdate;
+	static Timer::Ptr m_MembersCacheTimer;
 
 	static void RefreshMembersCache(void);
 };

lib/icinga/pluginnotificationtask.cpp

@@ -44,7 +44,7 @@ void PluginNotificationTask::ScriptFunc(const ScriptTask::Ptr& task, const vecto
 
 	Notification::Ptr notification = arguments[0];
 	Dictionary::Ptr macros = arguments[1];
-	NotificationType type = static_cast<NotificationType>(static_cast<int>(arguments[2]));
+//	NotificationType type = static_cast<NotificationType>(static_cast<int>(arguments[2]));
 
 	Value raw_command = notification->GetNotificationCommand();
 

lib/icinga/service-check.cpp

@@ -334,6 +334,8 @@ void Service::SetForceNextCheck(bool forced)
  */
 void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 {
+	bool reachable = IsReachable();
+
 	assert(!OwnsLock());
 	ObjectLock olock(this);
 
@@ -413,6 +415,8 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 
 	Service::UpdateStatistics(cr);
 
+	bool send_notification = hardChange && reachable && !IsInDowntime() && !IsAcknowledged();
+
 	olock.Unlock();
 
 	/* Flush the object so other instances see the service's
@@ -431,7 +435,7 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 
 	EndpointManager::GetInstance()->SendMulticastMessage(rm);
 
-	if (hardChange && IsReachable() && !IsInDowntime() && !IsAcknowledged())
+	if (send_notification)
 		RequestNotifications(recovery ? NotificationRecovery : NotificationProblem);
 }
 
@@ -521,10 +525,12 @@ bool Service::IsAllowedChecker(const String& checker) const
 void Service::BeginExecuteCheck(const function<void (void)>& callback)
 {
 	assert(!OwnsLock());
+
+	{
 		ObjectLock olock(this);
 
 		/* don't run another check if there is one pending */
-	if (m_CurrentTask) {
+		if (m_CheckRunning) {
 			olock.Unlock();
 
 			/* we need to call the callback anyway */
@@ -533,6 +539,9 @@ void Service::BeginExecuteCheck(const function<void (void)>& callback)
 			return;
 		}
 
+		m_CheckRunning = true;
+	}
+
 	/* keep track of scheduling info in case the check type doesn't provide its own information */
 	Dictionary::Ptr checkInfo = boost::make_shared<Dictionary>();
 	checkInfo->Set("schedule_start", GetNextCheck());
@@ -545,12 +554,12 @@ void Service::BeginExecuteCheck(const function<void (void)>& callback)
 
 	Host::Ptr host = GetHost();
 
-	olock.Unlock();
-
 	macroDicts.push_back(CalculateDynamicMacros());
 
+	if (host) {
 		macroDicts.push_back(host->GetMacros());
 		macroDicts.push_back(host->CalculateDynamicMacros());
+	}
 
 	IcingaApplication::Ptr app = IcingaApplication::GetInstance();
 	macroDicts.push_back(app->GetMacros());
@@ -570,7 +579,7 @@ void Service::BeginExecuteCheck(const function<void (void)>& callback)
 	ScriptTask::Ptr task = MakeMethodTask("check", arguments);
 
 	{
-		ObjectLock slock(this);
+		ObjectLock olock(this);
 		self->m_CurrentTask = task;
 	}
 
@@ -642,6 +651,7 @@ void Service::CheckCompletedHandler(const Dictionary::Ptr& checkInfo,
 	{
 		ObjectLock olock(this);
 		m_CurrentTask.reset();
+		m_CheckRunning = false;
 	}
 
 	/* figure out when the next check is for this service; the call to

lib/icinga/service-comment.cpp

@@ -25,7 +25,8 @@ int Service::m_NextCommentID = 1;
 boost::mutex Service::m_CommentMutex;
 map<int, String> Service::m_LegacyCommentsCache;
 map<String, Service::WeakPtr> Service::m_CommentsCache;
-bool Service::m_CommentsCacheValid = true;
+bool Service::m_CommentsCacheNeedsUpdate = false;
+Timer::Ptr Service::m_CommentsCacheTimer;
 Timer::Ptr Service::m_CommentsExpireTimer;
 
 /**
@@ -176,10 +177,17 @@ void Service::InvalidateCommentsCache(void)
 {
 	boost::mutex::scoped_lock lock(m_CommentMutex);
 
-	if (m_CommentsCacheValid)
+	if (m_CommentsCacheNeedsUpdate)
-		Utility::QueueAsyncCallback(boost::bind(&Service::RefreshCommentsCache));
+		return; /* Someone else has already requested a refresh. */
 
-	m_CommentsCacheValid = false;
+	if (!m_CommentsCacheTimer) {
+		m_CommentsCacheTimer = boost::make_shared<Timer>();
+		m_CommentsCacheTimer->SetInterval(0.5);
+		m_CommentsCacheTimer->OnTimerExpired.connect(boost::bind(&Service::RefreshCommentsCache));
+	}
+
+	m_CommentsCacheTimer->Start();
+	m_CommentsCacheNeedsUpdate = true;
 }
 
 /**
@@ -190,10 +198,9 @@ void Service::RefreshCommentsCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_CommentMutex);
 
-		if (m_CommentsCacheValid)
+		assert(m_CommentsCacheNeedsUpdate);
-			return;
+		m_CommentsCacheTimer->Stop();
-
+		m_CommentsCacheNeedsUpdate = false;
-		m_CommentsCacheValid = true;
 	}
 
 	map<int, String> newLegacyCommentsCache;

lib/icinga/service-downtime.cpp

@@ -25,7 +25,8 @@ int Service::m_NextDowntimeID = 1;
 boost::mutex Service::m_DowntimeMutex;
 map<int, String> Service::m_LegacyDowntimesCache;
 map<String, Service::WeakPtr> Service::m_DowntimesCache;
-bool Service::m_DowntimesCacheValid = true;
+bool Service::m_DowntimesCacheNeedsUpdate = false;
+Timer::Ptr Service::m_DowntimesCacheTimer;
 Timer::Ptr Service::m_DowntimesExpireTimer;
 
 /**
@@ -250,10 +251,17 @@ void Service::InvalidateDowntimesCache(void)
 {
 	boost::mutex::scoped_lock lock(m_DowntimeMutex);
 
-	if (m_DowntimesCacheValid)
+	if (m_DowntimesCacheNeedsUpdate)
-		Utility::QueueAsyncCallback(boost::bind(&Service::RefreshDowntimesCache));
+		return; /* Someone else has already requested a refresh. */
 
-	m_DowntimesCacheValid = false;
+	if (!m_DowntimesCacheTimer) {
+		m_DowntimesCacheTimer = boost::make_shared<Timer>();
+		m_DowntimesCacheTimer->SetInterval(0.5);
+		m_DowntimesCacheTimer->OnTimerExpired.connect(boost::bind(&Service::RefreshNotificationsCache));
+	}
+
+	m_DowntimesCacheTimer->Start();
+	m_DowntimesCacheNeedsUpdate = true;
 }
 
 /**
@@ -264,10 +272,9 @@ void Service::RefreshDowntimesCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_DowntimeMutex);
 
-		if (m_DowntimesCacheValid)
+		assert(m_DowntimesCacheNeedsUpdate);
-			return;
+		m_DowntimesCacheTimer->Stop();
-
+		m_DowntimesCacheNeedsUpdate = false;
-		m_DowntimesCacheValid = true;
 	}
 
 	map<int, String> newLegacyDowntimesCache;

lib/icinga/service-notification.cpp

@@ -23,14 +23,18 @@ using namespace icinga;
 
 boost::mutex Service::m_NotificationMutex;
 map<String, set<Notification::WeakPtr> > Service::m_NotificationsCache;
-bool Service::m_NotificationsCacheValid = true;
+bool Service::m_NotificationsCacheNeedsUpdate = false;
+Timer::Ptr Service::m_NotificationsCacheTimer;
 
 /**
  * @threadsafety Always.
  */
 void Service::RequestNotifications(NotificationType type)
 {
+	{
+		ObjectLock olock(this);
 		SetLastNotification(Utility::GetTime());
+	}
 
 	RequestMessage msg;
 	msg.SetMethod("icinga::SendNotifications");
@@ -83,10 +87,17 @@ void Service::InvalidateNotificationsCache(void)
 {
 	boost::mutex::scoped_lock lock(m_NotificationMutex);
 
-	if (m_NotificationsCacheValid)
+	if (m_NotificationsCacheNeedsUpdate)
-		Utility::QueueAsyncCallback(boost::bind(&Service::RefreshNotificationsCache));
+		return; /* Someone else has already requested a refresh. */
 
-	m_NotificationsCacheValid = false;
+	if (!m_NotificationsCacheTimer) {
+		m_NotificationsCacheTimer = boost::make_shared<Timer>();
+		m_NotificationsCacheTimer->SetInterval(0.5);
+		m_NotificationsCacheTimer->OnTimerExpired.connect(boost::bind(&Service::RefreshNotificationsCache));
+	}
+
+	m_NotificationsCacheTimer->Start();
+	m_NotificationsCacheNeedsUpdate = true;
 }
 
 /**
@@ -97,10 +108,9 @@ void Service::RefreshNotificationsCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_NotificationMutex);
 
-		if (m_NotificationsCacheValid)
+		assert(m_NotificationsCacheNeedsUpdate);
-			return;
+		m_NotificationsCacheTimer->Stop();
-
+		m_NotificationsCacheNeedsUpdate = false;
-		m_NotificationsCacheValid = true;
 	}
 
 	map<String, set<Notification::WeakPtr> > newNotificationsCache;

lib/icinga/service.cpp

@@ -24,7 +24,7 @@ using namespace icinga;
 REGISTER_TYPE(Service);
 
 Service::Service(const Dictionary::Ptr& serializedObject)
-	: DynamicObject(serializedObject)
+	: DynamicObject(serializedObject), m_CheckRunning(false)
 {
 
 	RegisterAttribute("display_name", Attribute_Config, &m_DisplayName);
@@ -106,9 +106,6 @@ Service::Ptr Service::GetByName(const String& name)
 {
 	DynamicObject::Ptr configObject = DynamicObject::GetObject("Service", name);
 
-	if (!configObject)
-		BOOST_THROW_EXCEPTION(invalid_argument("Service '" + name + "' does not exist."));
-
 	return dynamic_pointer_cast<Service>(configObject);
 }
 
@@ -250,6 +247,8 @@ bool Service::IsReachable(void) const
  */
 AcknowledgementType Service::GetAcknowledgement(void)
 {
+	assert(OwnsLock());
+
 	if (m_Acknowledgement.IsEmpty())
 		return AcknowledgementNone;
 

lib/icinga/service.h

@@ -284,6 +284,7 @@ private:
 	Attribute<bool> m_ForceNextCheck;
 
 	ScriptTask::Ptr m_CurrentTask;
+	bool m_CheckRunning;
 	long m_SchedulingOffset;
 
 	void CheckCompletedHandler(const Dictionary::Ptr& checkInfo,
@@ -297,7 +298,8 @@ private:
 	static boost::mutex m_DowntimeMutex;
 	static map<int, String> m_LegacyDowntimesCache;
 	static map<String, Service::WeakPtr> m_DowntimesCache;
-	static bool m_DowntimesCacheValid;
+	static bool m_DowntimesCacheNeedsUpdate;
+	static Timer::Ptr m_DowntimesCacheTimer;
 	static Timer::Ptr m_DowntimesExpireTimer;
 
 	static void DowntimesExpireTimerHandler(void);
@@ -314,7 +316,8 @@ private:
 	static boost::mutex m_CommentMutex;
 	static map<int, String> m_LegacyCommentsCache;
 	static map<String, Service::WeakPtr> m_CommentsCache;
-	static bool m_CommentsCacheValid;
+	static bool m_CommentsCacheNeedsUpdate;
+	static Timer::Ptr m_CommentsCacheTimer;
 	static Timer::Ptr m_CommentsExpireTimer;
 
 	static void CommentsExpireTimerHandler(void);
@@ -331,7 +334,8 @@ private:
 
 	static boost::mutex m_NotificationMutex;
 	static map<String, set<Notification::WeakPtr> > m_NotificationsCache;
-	static bool m_NotificationsCacheValid;
+	static bool m_NotificationsCacheNeedsUpdate;
+	static Timer::Ptr m_NotificationsCacheTimer;
 
 	static void RefreshNotificationsCache(void);
 };

lib/icinga/servicegroup.cpp

@@ -23,7 +23,8 @@ using namespace icinga;
 
 boost::mutex ServiceGroup::m_Mutex;
 map<String, vector<Service::WeakPtr> > ServiceGroup::m_MembersCache;
-bool ServiceGroup::m_MembersCacheValid = true;
+bool ServiceGroup::m_MembersCacheNeedsUpdate = false;
+Timer::Ptr ServiceGroup::m_MembersCacheTimer;
 
 REGISTER_TYPE(ServiceGroup);
 
@@ -120,10 +121,17 @@ void ServiceGroup::InvalidateMembersCache(void)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);
 
-	if (m_MembersCacheValid)
+	if (m_MembersCacheNeedsUpdate)
-		Utility::QueueAsyncCallback(boost::bind(&ServiceGroup::RefreshMembersCache));
+		return; /* Someone else has already requested a refresh. */
 
-	m_MembersCacheValid = false;
+	if (!m_MembersCacheTimer) {
+		m_MembersCacheTimer = boost::make_shared<Timer>();
+		m_MembersCacheTimer->SetInterval(0.5);
+		m_MembersCacheTimer->OnTimerExpired.connect(boost::bind(&ServiceGroup::RefreshMembersCache));
+	}
+
+	m_MembersCacheTimer->Start();
+	m_MembersCacheNeedsUpdate = true;
 }
 
 /**
@@ -134,10 +142,9 @@ void ServiceGroup::RefreshMembersCache(void)
 	{
 		boost::mutex::scoped_lock lock(m_Mutex);
 
-		if (m_MembersCacheValid)
+		assert(m_MembersCacheNeedsUpdate);
-			return;
+		m_MembersCacheTimer->Stop();
-
+		m_MembersCacheNeedsUpdate = false;
-		m_MembersCacheValid = true;
 	}
 
 	map<String, vector<Service::WeakPtr> > newMembersCache;

lib/icinga/servicegroup.h

@@ -57,7 +57,8 @@ private:
 
 	static boost::mutex m_Mutex;
 	static map<String, vector<Service::WeakPtr> > m_MembersCache;
-	static bool m_MembersCacheValid;
+	static bool m_MembersCacheNeedsUpdate;
+	static Timer::Ptr m_MembersCacheTimer;
 
 	static void RefreshMembersCache(void);
 };