Fine-grained locks (WIP, Part 9).

components/checker/checkercomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(checker, CheckerComponent);
+REGISTER_COMPONENT("checker", CheckerComponent);
 
 void CheckerComponent::Start(void)
 {
@@ -59,12 +59,9 @@ void CheckerComponent::Stop(void)
 
 void CheckerComponent::CheckThreadProc(void)
 {
+	boost::mutex::scoped_lock lock(m_Mutex);
+
 	for (;;) {
-		vector<Service::Ptr> services;
-		Service::Ptr service;
-
-		boost::mutex::scoped_lock lock(m_Mutex);
-
 		typedef nth_index<ServiceSet, 1>::type CheckTimeView;
 		CheckTimeView& idx = boost::get<1>(m_IdleServices);
 
@@ -75,7 +72,7 @@ void CheckerComponent::CheckThreadProc(void)
 			break;
 
 		CheckTimeView::iterator it = idx.begin();
-		service = it->lock();
+		Service::Ptr service = it->lock();
 
 		if (!service) {
 			idx.erase(it);
@@ -131,20 +128,16 @@ void CheckerComponent::CheckThreadProc(void)
 		m_IdleServices.erase(service);
 		m_PendingServices.insert(service);
 
-		double rwait = service->GetNextCheck() - Utility::GetTime();
-
-		if (rwait < -5)
-			Logger::Write(LogWarning, "checker", "Check delayed: " + Convert::ToString(-rwait));
-
 		try {
-			service->BeginExecuteCheck(boost::bind(&CheckerComponent::CheckCompletedHandler, this, service));
+			olock.Unlock();
+			Service::BeginExecuteCheck(service, boost::bind(&CheckerComponent::CheckCompletedHandler, this, service));
 		} catch (const exception& ex) {
+			olock.Lock();
 			Logger::Write(LogCritical, "checker", "Exception occured while checking service '" + service->GetName() + "': " + diagnostic_information(ex));
 		}
 	}
 }
 
-
 void CheckerComponent::CheckCompletedHandler(const Service::Ptr& service)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);
@@ -217,4 +210,3 @@ void CheckerComponent::NextCheckChangedHandler(const Service::Ptr& service)
 	idx.insert(service);
 	m_CV.notify_all();
 }
-

components/compat/compatcomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(compat, CompatComponent);
+REGISTER_COMPONENT("compat", CompatComponent);
 
 /**
  * Hint: The reason why we're using "\n" rather than std::endl is because
@@ -91,7 +91,7 @@ String CompatComponent::GetCommandPath(void) const
 void CompatComponent::Start(void)
 {
 	m_StatusTimer = boost::make_shared<Timer>();
-	m_StatusTimer->SetInterval(60);
+	m_StatusTimer->SetInterval(15);
 	m_StatusTimer->OnTimerExpired.connect(boost::bind(&CompatComponent::StatusTimerHandler, this));
 	m_StatusTimer->Start();
 	m_StatusTimer->Reschedule(0);
@@ -323,11 +323,10 @@ void CompatComponent::DumpServiceStatusAttrs(ofstream& fp, const Service::Ptr& s
 {
 	String output;
 	String perfdata;
-	double schedule_start = -1, schedule_end = -1;
+	double schedule_end = -1;
-	double execution_start = -1, execution_end = -1;
 
 	Dictionary::Ptr cr;
-	int state;
+	int state, state_type;
 	Host::Ptr host;
 
 	{
@@ -335,21 +334,16 @@ void CompatComponent::DumpServiceStatusAttrs(ofstream& fp, const Service::Ptr& s
 
 		cr = service->GetLastCheckResult();
 		state = service->GetState();
+		state_type = service->GetStateType();
 		host = service->GetHost();
 	}
 
 	if (cr) {
 		output = cr->Get("output");
-		schedule_start = cr->Get("schedule_start");
 		schedule_end = cr->Get("schedule_end");
-		execution_start = cr->Get("execution_start");
-		execution_end = cr->Get("execution_end");
 		perfdata = cr->Get("performance_data_raw");
 	}
 
-	double execution_time = (execution_end - execution_start);
-	double latency = (schedule_end - schedule_start) - execution_time;
-
 	if (state > StateUnknown)
 		state = StateUnknown;
 
@@ -370,10 +364,10 @@ void CompatComponent::DumpServiceStatusAttrs(ofstream& fp, const Service::Ptr& s
 		   << "\t" << "retry_interval=" << service->GetRetryInterval() / 60.0 << "\n"
 		   << "\t" << "has_been_checked=" << (service->GetLastCheckResult() ? 1 : 0) << "\n"
 		   << "\t" << "should_be_scheduled=1" << "\n"
-		   << "\t" << "check_execution_time=" << execution_time << "\n"
+		   << "\t" << "check_execution_time=" << Service::CalculateExecutionTime(cr) << "\n"
-		   << "\t" << "check_latency=" << latency << "\n"
+		   << "\t" << "check_latency=" << Service::CalculateLatency(cr) << "\n"
 		   << "\t" << "current_state=" << state << "\n"
-		   << "\t" << "state_type=" << service->GetStateType() << "\n"
+		   << "\t" << "state_type=" << state_type << "\n"
 		   << "\t" << "plugin_output=" << output << "\n"
 		   << "\t" << "performance_data=" << perfdata << "\n"
 		   << "\t" << "last_check=" << schedule_end << "\n"
@@ -518,7 +512,8 @@ void CompatComponent::StatusTimerHandler(void)
 		 << "\t" << "passive_host_checks_enabled=0" << "\n"
 		 << "\t" << "check_service_freshness=0" << "\n"
 		 << "\t" << "check_host_freshness=0" << "\n"
-		 << "\t" << "enable_flap_detection=1" << "\n"
+		 << "\t" << "enable_notifications=1" << "\n"
+		 << "\t" << "enable_flap_detection=0" << "\n"
 		 << "\t" << "enable_failure_prediction=0" << "\n"
 		 << "\t" << "active_scheduled_service_check_stats=" << CIB::GetActiveChecksStatistics(60) << "," << CIB::GetActiveChecksStatistics(5 * 60) << "," << CIB::GetActiveChecksStatistics(15 * 60) << "\n"
 		 << "\t" << "passive_service_check_stats=" << CIB::GetPassiveChecksStatistics(60) << "," << CIB::GetPassiveChecksStatistics(5 * 60) << "," << CIB::GetPassiveChecksStatistics(15 * 60) << "\n"

components/compatido/compatidocomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(compatido, CompatIdoComponent);
+REGISTER_COMPONENT("compatido", CompatIdoComponent);
 
 const String CompatIdoComponent::DefaultSocketAddress = "127.0.0.1";
 const String CompatIdoComponent::DefaultSocketPort = "5668";

components/delegation/delegationcomponent.cpp

@@ -22,7 +22,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(delegation, DelegationComponent);
+REGISTER_COMPONENT("delegation", DelegationComponent);
 
 void DelegationComponent::Start(void)
 {

components/demo/democomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(demo, DemoComponent);
+REGISTER_COMPONENT("demo", DemoComponent);
 
 /**
  * Starts the component.

components/notification/notificationcomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(notification, NotificationComponent);
+REGISTER_COMPONENT("notification", NotificationComponent);
 
 /**
  * Starts the component.

components/replication/replicationcomponent.cpp

@@ -21,7 +21,7 @@
 
 using namespace icinga;
 
-EXPORT_COMPONENT(replication, ReplicationComponent);
+REGISTER_COMPONENT("replication", ReplicationComponent);
 
 /**
  * Starts the component.

icinga-app/icinga.cpp

@@ -210,7 +210,7 @@ int main(int argc, char **argv)
 
 	Component::AddSearchDir(Application::GetPkgLibDir());
 
-	Utility::LoadIcingaLibrary("icinga", false);
+	(void) Utility::LoadIcingaLibrary("icinga", false);
 
 	if (g_AppParams.count("library")) {
 		BOOST_FOREACH(const String& libraryName, g_AppParams["library"].as<vector<String> >()) {

lib/base/Makefile.am

@@ -34,6 +34,8 @@ libbase_la_SOURCES =  \
 	netstring.h \
 	object.cpp \
 	object.h \
+	objectlock.cpp \
+	objectlock.h \
 	process.cpp \
 	process-unix.cpp \
 	process-windows.cpp \

lib/base/application.cpp

@@ -120,8 +120,12 @@ void Application::ProfileTimerHandler(void)
 
 void Application::ShutdownTimerHandler(void)
 {
-	if (m_ShuttingDown)
+	if (m_ShuttingDown) {
+		Application::GetInstance()->OnShutdown();
+		DynamicObject::DeactivateObjects();
 		GetEQ().Stop();
+		m_ShuttingDown = false;
+	}
 }
 
 /**
@@ -424,8 +428,6 @@ int Application::Run(void)
 
 	result = Main();
 
-	DynamicObject::DeactivateObjects();
-
 	return result;
 }
 

lib/base/application.h

@@ -84,6 +84,8 @@ public:
 protected:
 	void RunEventLoop(void) const;
 
+	virtual void OnShutdown(void) = 0;
+
 private:
 	static Application *m_Instance; /**< The application instance. */
 

lib/base/asynctask.h

@@ -66,7 +66,7 @@ public:
 	void Start(const CompletionCallback& completionCallback = CompletionCallback())
 	{
 		m_CompletionCallback = completionCallback;
-		Utility::QueueAsyncCallback(boost::bind(&AsyncTask<TClass, TResult>::Run, this));
+		Utility::QueueAsyncCallback(boost::bind(&AsyncTask<TClass, TResult>::RunInternal, this));
 	}
 
 	/**
@@ -166,6 +166,18 @@ private:
 			Utility::QueueAsyncCallback(boost::bind(callback, GetSelf()));
 	}
 
+	/**
+	 * Calls the Run() method and catches exceptions.
+	 */
+	void RunInternal(void)
+	{
+		try {
+			Run();
+		} catch (const exception& ex) {
+			FinishException(boost::current_exception());
+		}
+	}
+
 	mutable boost::mutex m_Mutex;
 	boost::condition_variable m_CV;
 	CompletionCallback m_CompletionCallback; /**< The completion callback. */

lib/base/component.cpp

@@ -23,6 +23,8 @@ using namespace icinga;
 
 REGISTER_TYPE(Component, NULL);
 
+map<String, Component::Factory> Component::m_Factories;
+
 /**
  * Constructor for the component class.
  */
@@ -32,51 +34,20 @@ Component::Component(const Dictionary::Ptr& properties)
 	if (!IsLocal())
 		BOOST_THROW_EXCEPTION(runtime_error("Component objects must be local."));
 
-#ifdef _WIN32
-	HMODULE
-#else /* _WIN32 */
-	lt_dlhandle
-#endif /* _WIN32 */
-	hModule;
-
 	Logger::Write(LogInformation, "base", "Loading component '" + GetName() + "'");
 
-	hModule = Utility::LoadIcingaLibrary(GetName(), true);
+	(void) Utility::LoadIcingaLibrary(GetName(), true);
 
-	CreateComponentFunction pCreateComponent;
+	map<String, Factory>::iterator it;
+	it = m_Factories.find(GetName());
 
-#ifdef _WIN32
+	if (it == m_Factories.end())
-	pCreateComponent = reinterpret_cast<CreateComponentFunction>(GetProcAddress(hModule,
+		BOOST_THROW_EXCEPTION(invalid_argument("Unknown component: " + GetName()));
-	    "CreateComponent"));
-#else /* _WIN32 */
-#	ifdef __GNUC__
-	/* suppress compiler warning for void * cast */
-	__extension__
-#	endif
-	pCreateComponent = reinterpret_cast<CreateComponentFunction>(lt_dlsym(hModule,
-	    "CreateComponent"));
-#endif /* _WIN32 */
 
-	IComponent::Ptr impl;
+	IComponent::Ptr impl = it->second();
 
-	try {
+	if (!impl)
-		if (pCreateComponent == NULL)
+		BOOST_THROW_EXCEPTION(runtime_error("Component factory returned NULL."));
-			BOOST_THROW_EXCEPTION(runtime_error("Loadable module does not contain "
-			    "CreateComponent function"));
-
-		/* pCreateComponent returns a raw pointer which we must wrap in a shared_ptr */
-		impl = IComponent::Ptr(pCreateComponent());
-
-		if (!impl)
-			BOOST_THROW_EXCEPTION(runtime_error("CreateComponent function returned NULL."));
-	} catch (...) {
-#ifdef _WIN32
-		FreeLibrary(hModule);
-#else /* _WIN32 */
-		lt_dlclose(hModule);
-#endif /* _WIN32 */
-		throw;
-	}
 
 	m_Impl = impl;
 }
@@ -142,3 +113,11 @@ void IComponent::Stop(void)
 {
 	/* Nothing to do in the default implementation. */
 }
+
+/**
+ * Registers a component factory.
+ */
+void Component::Register(const String& name, const Component::Factory& factory)
+{
+	m_Factories[name] = factory;
+}

lib/base/component.h

@@ -59,6 +59,8 @@ public:
 	typedef shared_ptr<Component> Ptr;
 	typedef weak_ptr<Component> WeakPtr;
 
+	typedef function<IComponent::Ptr (void)> Factory;
+
 	Component(const Dictionary::Ptr& properties);
 	~Component(void);
 
@@ -66,30 +68,43 @@ public:
 
 	static void AddSearchDir(const String& componentDirectory);
 
+	static void Register(const String& name, const Factory& factory);
+
 private:
 	IComponent::Ptr m_Impl; /**< The implementation object for this
 				     component. */
+
+	static map<String, Factory> m_Factories;
 };
 
-typedef IComponent *(*CreateComponentFunction)(void);
+/**
-
+ * Helper class for registering Component implementation classes.
-#ifdef _WIN32
+ *
-#	define SYM_CREATECOMPONENT(component) CreateComponent
+ * @ingroup base
-#else /* _WIN32 */
+ */
-#	define SYM_CREATECOMPONENT(component) component ## _LTX_CreateComponent
+class RegisterComponentHelper
-#endif /* _WIN32 */
+{
+public:
+	RegisterComponentHelper(const String& name, const Component::Factory& factory)
+	{
+		Component::Register(name, factory);
+	}
+};
 
 /**
- * Implements the loader function for a component.
+ * Factory function for IComponent-based classes.
  *
- * @param component The name of the component.
+ * @ingroup base
- * @param klass The component class.
  */
-#define EXPORT_COMPONENT(component, klass) \
+template<typename T>
-	extern "C" I2_EXPORT icinga::IComponent *SYM_CREATECOMPONENT(component)(void) \
+IComponent::Ptr ComponentFactory(void)
-	{								\
+{
-		return new klass();					\
+	return boost::make_shared<T>();
-	}
+}
+
+
+#define REGISTER_COMPONENT(name, klass) \
+	static RegisterComponentHelper g_RegisterSF_ ## type(name, ComponentFactory<klass>)
 
 }
 

lib/base/dictionary.cpp

@@ -54,6 +54,13 @@ struct DictionaryKeyLessComparer
 	}
 };
 
+/**
+ * Constructor for the Dictionary class.
+ */
+Dictionary::Dictionary(void)
+	: m_Sealed(false)
+{ }
+
 /**
  * Restrieves a value from a dictionary.
  *
@@ -98,6 +105,8 @@ void Dictionary::Set(const String& key, const Value& value)
 {
 	ObjectLock olock(this);
 
+	assert(!m_Sealed);
+
 	if (value.IsEmpty()) {
 		Remove(key);
 		return;
@@ -213,6 +222,15 @@ void Dictionary::Remove(Dictionary::Iterator it)
 	m_Data.erase(it);
 }
 
+/**
+ * Marks the dictionary as read-only. Attempting to modify a sealed
+ * dictionary is an error.
+ */
+void Dictionary::Seal(void)
+{
+	m_Sealed = true;
+}
+
 /**
  * Makes a shallow copy of a dictionary.
  *

lib/base/dictionary.h

@@ -39,11 +39,14 @@ public:
 	 */
 	typedef map<String, Value>::iterator Iterator;
 
+	Dictionary(void);
+
 	Value Get(const char *key) const;
 	Value Get(const String& key) const;
 	void Set(const String& key, const Value& value);
 	String Add(const Value& value);
 	bool Contains(const String& key) const;
+	void Seal(void);
 
 	Iterator Begin(void);
 	Iterator End(void);
@@ -60,6 +63,7 @@ public:
 
 private:
 	map<String, Value> m_Data; /**< The data for the dictionary. */
+	bool m_Sealed; /**< Whether the dictionary is read-only. */
 };
 
 inline Dictionary::Iterator range_begin(Dictionary::Ptr x)

lib/base/dynamicobject.cpp

@@ -122,6 +122,8 @@ Dictionary::Ptr DynamicObject::BuildUpdate(double sinceTx, int attributeTypes) c
 		attrs->Set(it->first, attr);
 	}
 
+	attrs->Seal();
+
 	Dictionary::Ptr update = boost::make_shared<Dictionary>();
 	update->Set("attrs", attrs);
 
@@ -130,6 +132,8 @@ Dictionary::Ptr DynamicObject::BuildUpdate(double sinceTx, int attributeTypes) c
 	else if (attrs->GetLength() == 0)
 		return Dictionary::Ptr();
 
+	update->Seal();
+
 	return update;
 }
 
@@ -243,8 +247,12 @@ void DynamicObject::InternalSetAttribute(const String& name, const Value& data,
 		 * object to the list of modified objects later on if we can't
 		 * do it here. */
 
-		boost::mutex::scoped_lock lock(m_TransactionMutex);
+		DynamicObject::Ptr self = GetSelf();
-		m_ModifiedObjects.insert(GetSelf());
+
+		{
+			boost::mutex::scoped_lock lock(m_TransactionMutex);
+			m_ModifiedObjects.insert(self);
+		}
 	}
 
 	/* Use insert() rather than [] so we don't overwrite
@@ -496,7 +504,7 @@ void DynamicObject::RestoreObjects(const String& filename)
 
 	stringstream msgbuf;
 	msgbuf << "Restored " << restored << " objects";
-	Logger::Write(LogDebug, "base", msgbuf.str());
+	Logger::Write(LogInformation, "base", msgbuf.str());
 }
 
 void DynamicObject::DeactivateObjects(void)

lib/base/eventqueue.cpp

@@ -81,7 +81,7 @@ void EventQueue::QueueThreadProc(void)
 			while (m_Events.empty() && !m_Stopped)
 				m_CV.wait(lock);
 
-			if (m_Stopped)
+			if (m_Events.empty() && m_Stopped)
 				break;
 
 			events.swap(m_Events);
@@ -94,7 +94,7 @@ void EventQueue::QueueThreadProc(void)
 
 			double et = Utility::GetTime();
 
-			if (et - st > 1.0) {
+			if (et - st > 0.25) {
 				stringstream msgbuf;
 				msgbuf << "Event call took " << et - st << " seconds.";
 				Logger::Write(LogWarning, "base", msgbuf.str());
@@ -118,7 +118,7 @@ void EventQueue::Post(const EventQueue::Callback& callback)
 	int pending = m_Events.size();
 	double now = Utility::GetTime();
 	if (pending > 1000 && now - m_LastReport > 5) {
-		Logger::Write(LogWarning, "base", "More than 1000 pending events: " + Convert::ToString(pending));
+		Logger::Write(LogCritical, "base", "More than 1000 pending events: " + Convert::ToString(pending));
 		m_LastReport = now;
 	}
 }

lib/base/i2-base.h

@@ -190,6 +190,7 @@ namespace signals2 = boost::signals2;
 #include "qstring.h"
 #include "utility.h"
 #include "object.h"
+#include "objectlock.h"
 #include "exception.h"
 #include "eventqueue.h"
 #include "value.h"

lib/base/object.h

@@ -106,38 +106,6 @@ private:
 	mutable recursive_mutex m_Mutex;
 };
 
-/**
- * A scoped lock for Objects.
- */
-struct ObjectLock {
-public:
-	ObjectLock(void)
-		: m_Lock()
-	{ }
-
-	ObjectLock(const Object::Ptr& object)
-		: m_Lock()
-	{
-		if (object)
-			m_Lock = recursive_mutex::scoped_lock(object->GetMutex());
-	}
-
-	ObjectLock(const Object *object)
-		: m_Lock()
-	{
-		if (object)
-			m_Lock = recursive_mutex::scoped_lock(object->GetMutex());
-	}
-
-	void Unlock(void)
-	{
-		m_Lock = recursive_mutex::scoped_lock();
-	}
-
-private:
-	recursive_mutex::scoped_lock m_Lock;
-};
-
 /**
  * Compares a weak pointer with a raw pointer.
  *

lib/base/process-unix.cpp

@@ -24,6 +24,7 @@
 using namespace icinga;
 
 int Process::m_TaskFd;
+Timer::Ptr Process::m_StatusTimer;
 extern char **environ;
 
 void Process::Initialize(void)
@@ -37,7 +38,7 @@ void Process::Initialize(void)
 	if (pipe(fds) < 0)
 		BOOST_THROW_EXCEPTION(PosixException("pipe() failed.", errno));
 
-	/* Don't bother setting fds[1] to clo-exec as we'll only
+	/* Don't bother setting fds[0] to clo-exec as we'll only
 	 * use it in the following dup() call. */
 
 	Utility::SetCloExec(fds[1]);
@@ -59,6 +60,11 @@ void Process::Initialize(void)
 	}
 
 	(void) close(fds[0]);
+
+	m_StatusTimer = boost::make_shared<Timer>();
+	m_StatusTimer->OnTimerExpired.connect(boost::bind(&Process::StatusTimerHandler));
+	m_StatusTimer->SetInterval(5);
+	m_StatusTimer->Start();
 }
 
 void Process::WorkerThreadProc(int taskFd)
@@ -314,4 +320,12 @@ bool Process::RunTask(void)
 	return false;
 }
 
+void Process::StatusTimerHandler(void)
+{
+	boost::mutex::scoped_lock lock(m_Mutex);
+	if (m_Tasks.size() > 50)
+		Logger::Write(LogCritical, "base", "More than 50 waiting Process tasks: " +
+		    Convert::ToString(m_Tasks.size()));
+}
+
 #endif /* _WIN32 */

lib/base/process.cpp

@@ -24,7 +24,6 @@ using namespace icinga;
 boost::once_flag Process::m_ThreadOnce = BOOST_ONCE_INIT;
 boost::mutex Process::m_Mutex;
 deque<Process::Ptr> Process::m_Tasks;
-double Process::m_LastReport = 0;
 
 Process::Process(const vector<String>& arguments, const Dictionary::Ptr& extraEnvironment)
 	: AsyncTask<Process, ProcessResult>(), m_Arguments(arguments), m_ExtraEnvironment(extraEnvironment)
@@ -66,18 +65,9 @@ vector<String> Process::SplitCommand(const Value& command)
 
 void Process::Run(void)
 {
-	int count;
-
 	{
 		boost::mutex::scoped_lock lock(m_Mutex);
 		m_Tasks.push_back(GetSelf());
-		count = m_Tasks.size();
-	}
-
-	if (count > 50 && Utility::GetTime() - m_LastReport > 5) {
-		Logger::Write(LogInformation, "base", "More than 50 pending Process tasks: " +
-		    Convert::ToString(count));
-		m_LastReport = Utility::GetTime();
 	}
 
 	NotifyWorker();

lib/base/process.h

@@ -69,10 +69,11 @@ private:
 	virtual void Run(void);
 
 	static boost::mutex m_Mutex;
-	static double m_LastReport;
 	static deque<Process::Ptr> m_Tasks;
 #ifndef _WIN32
 	static int m_TaskFd;
+
+	static Timer::Ptr m_StatusTimer;
 #endif /* _WIN32 */
 
 	static void NotifyWorker(void);
@@ -83,6 +84,8 @@ private:
 	static void WorkerThreadProc(void);
 #else /* _WIN32 */
 	static void WorkerThreadProc(int taskFd);
+
+	static void StatusTimerHandler(void);
 #endif /* _WIN32 */
 
 	void InitTask(void);

lib/base/value.cpp

@@ -54,6 +54,38 @@ bool Value::IsObject(void) const
 	return !IsEmpty() && (m_Value.type() == typeid(Object::Ptr));
 }
 
+Value::operator double(void) const
+{
+	if (m_Value.type() != typeid(double)) {
+		return boost::lexical_cast<double>(m_Value);
+	} else {
+		return boost::get<double>(m_Value);
+	}
+}
+
+Value::operator String(void) const
+{
+	Object *object;
+	double integral, fractional;
+
+	switch (GetType()) {
+		case ValueEmpty:
+			return String();
+		case ValueNumber:
+			fractional = modf(boost::get<double>(m_Value), &integral);
+
+			if (fractional != 0)
+				return boost::lexical_cast<String>(m_Value);
+			else
+				return boost::lexical_cast<String>((long)integral);
+		case ValueString:
+			return boost::get<String>(m_Value);
+		case ValueObject:
+			object = boost::get<Object::Ptr>(m_Value).get();
+			return "Object of type '" + Utility::GetTypeName(typeid(*object)) + "'";
+	}
+}
+
 /**
  * Converts a JSON object into a variant.
  *

lib/base/value.h

@@ -85,27 +85,8 @@ public:
 		m_Value = object;
 	}
 
-	operator double(void) const
+	operator double(void) const;
-	{
+	operator String(void) const;
-		if (m_Value.type() != typeid(double)) {
-			return boost::lexical_cast<double>(m_Value);
-		} else {
-			return boost::get<double>(m_Value);
-		}
-	}
-
-	operator String(void) const
-	{
-		if (IsEmpty())
-			return String();
-
-		if (m_Value.type() != typeid(String)) {
-			String result = boost::lexical_cast<String>(m_Value);
-			m_Value = result;
-		}
-
-		return boost::get<String>(m_Value);
-	}
 
 	template<typename T>
 	operator shared_ptr<T>(void) const

lib/config/config_lexer.cc

@@ -386,7 +386,7 @@ static yyconst flex_int16_t yy_accept[193] =
        54,   49,   42,   42,   42,   42,   42,   42,   42,   42,
        42,   42,   42,   54,   18,   19,   12,    3,    2,   55,
        15,   15,   21,    0,    0,    0,    0,    0,   42,   52,
-       50,   48,   51,   16,    0,   53,    0,   45,   46,   47,
+       50,   48,   51,   16,   20,   53,    0,   45,   46,   47,
         0,   43,   42,   42,   42,   42,   42,   42,   42,   42,
        42,   42,   42,   42,   42,   42,    0,   18,   17,   12,
        11,    4,    5,    9,   10,    6,    8,    7,    0,    0,
@@ -793,6 +793,10 @@ int yyget_lineno (yyscan_t yyscanner );
 
 void yyset_lineno (int line_number ,yyscan_t yyscanner );
 
+int yyget_column  (yyscan_t yyscanner );
+
+void yyset_column (int column_no ,yyscan_t yyscanner );
+
 YYSTYPE * yyget_lval (yyscan_t yyscanner );
 
 void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
@@ -948,7 +952,7 @@ YY_DECL
 
 	lex_buf string_buf;
 
-#line 952 "config_lexer.cc"
+#line 956 "config_lexer.cc"
 
     yylval = yylval_param;
 
@@ -1377,7 +1381,7 @@ YY_RULE_SETUP
 #line 216 "config_lexer.ll"
 ECHO;
 	YY_BREAK
-#line 1381 "config_lexer.cc"
+#line 1385 "config_lexer.cc"
 case YY_STATE_EOF(INITIAL):
 case YY_STATE_EOF(C_COMMENT):
 case YY_STATE_EOF(STRING):

lib/config/config_lexer.ll

@@ -173,7 +173,7 @@ static void lb_append_char(lex_buf *lb, char new_char)
 "*"				/* ignore star */
 }
 
-\/\/[^\n]+			/* ignore C++-style comments */
+\/\/[^\n]*			/* ignore C++-style comments */
 [ \t\r\n]+			/* ignore whitespace */
 
 <INITIAL>{

lib/icinga/host.cpp

@@ -288,6 +288,8 @@ void Host::UpdateSlaveServices(void)
 		}
 	}
 
+	newServices->Seal();
+
 	Set("slave_services", newServices);
 }
 
@@ -479,23 +481,69 @@ set<Service::Ptr> Host::GetParentServices(void) const
 	return parents;
 }
 
-Dictionary::Ptr Host::CalculateDynamicMacros(void) const
+Dictionary::Ptr Host::CalculateDynamicMacros(const Host::Ptr& self)
 {
 	Dictionary::Ptr macros = boost::make_shared<Dictionary>();
 
-	macros->Set("HOSTNAME", GetName());
+	Service::Ptr hc;
-	macros->Set("HOSTALIAS", GetName());
-	macros->Set("HOSTDISPLAYNAME", GetDisplayName());
-	macros->Set("HOSTSTATE", "DERP");
 
-	Service::Ptr hostcheck = GetHostCheckService();
+	{
+		ObjectLock olock(self);
 
-	if (hostcheck) {
+		macros->Set("HOSTNAME", self->GetName());
-		macros->Set("HOSTSTATEID", 99);
+		macros->Set("HOSTDISPLAYNAME", self->GetDisplayName());
-		macros->Set("HOSTSTATETYPE", Service::StateTypeToString(hostcheck->GetStateType()));
+		macros->Set("HOSTALIAS", self->GetName());
-		macros->Set("HOSTATTEMPT", hostcheck->GetCurrentCheckAttempt());
+
-		macros->Set("MAXHOSTATTEMPT", hostcheck->GetMaxCheckAttempts());
+		hc = self->GetHostCheckService();
 	}
 
+	bool reachable = Host::IsReachable(self);
+
+	Dictionary::Ptr cr;
+
+	if (hc) {
+		ObjectLock olock(hc);
+
+		String state;
+		int stateid;
+
+		switch (hc->GetState()) {
+			case StateOK:
+			case StateWarning:
+				state = "UP";
+				stateid = 0;
+				break;
+			default:
+				state = "DOWN";
+				stateid = 1;
+				break;
+		}
+
+		if (!reachable) {
+			state = "UNREACHABLE";
+			stateid = 2;
+		}
+
+		macros->Set("HOSTSTATE", state);
+		macros->Set("HOSTSTATEID", stateid);
+		macros->Set("HOSTSTATETYPE", Service::StateTypeToString(hc->GetStateType()));
+		macros->Set("HOSTATTEMPT", hc->GetCurrentCheckAttempt());
+		macros->Set("MAXHOSTATTEMPT", hc->GetMaxCheckAttempts());
+
+		cr = hc->GetLastCheckResult();
+	}
+
+	if (cr) {
+		macros->Set("HOSTLATENCY", Service::CalculateLatency(cr));
+		macros->Set("HOSTEXECUTIONTIME", Service::CalculateExecutionTime(cr));
+
+		ObjectLock olock(cr);
+
+		macros->Set("HOSTOUTPUT", cr->Get("output"));
+		macros->Set("HOSTPERFDATA", cr->Get("performance_data_raw"));
+	}
+
+	macros->Seal();
+
 	return macros;
 }

lib/icinga/host.h

@@ -50,7 +50,7 @@ public:
 	Dictionary::Ptr GetServiceDependencies(void) const;
 	String GetHostCheck(void) const;
 
-	Dictionary::Ptr CalculateDynamicMacros(void) const;
+	static Dictionary::Ptr CalculateDynamicMacros(const Host::Ptr& self);
 
 	shared_ptr<Service> GetHostCheckService(void) const;
 	set<Host::Ptr> GetParentHosts(void) const;

lib/icinga/icingaapplication.cpp

@@ -72,14 +72,20 @@ int IcingaApplication::Main(void)
 
 	RunEventLoop();
 
-	DumpProgramState();
-
 	Logger::Write(LogInformation, "icinga", "Icinga has shut down.");
 
 	return EXIT_SUCCESS;
 }
 
-void IcingaApplication::DumpProgramState(void) {
+void IcingaApplication::OnShutdown(void)
+{
+	m_RetentionTimer->Stop();
+
+	DumpProgramState();
+}
+
+void IcingaApplication::DumpProgramState(void)
+{
 	DynamicObject::DumpObjects(GetStatePath());
 }
 
@@ -142,3 +148,12 @@ shared_ptr<SSL_CTX> IcingaApplication::GetSSLContext(void) const
 {
 	return m_SSLContext;
 }
+
+Dictionary::Ptr IcingaApplication::CalculateDynamicMacros(const IcingaApplication::Ptr& self)
+{
+	Dictionary::Ptr macros = boost::make_shared<Dictionary>();
+
+	macros->Set("TIMET", (long)Utility::GetTime());
+
+	return macros;
+}

lib/icinga/icingaapplication.h

@@ -51,6 +51,8 @@ public:
 
 	double GetStartTime(void) const;
 
+	static Dictionary::Ptr CalculateDynamicMacros(const IcingaApplication::Ptr& self);
+
 private:
 	shared_ptr<SSL_CTX> m_SSLContext;
 
@@ -59,6 +61,8 @@ private:
 	Timer::Ptr m_RetentionTimer;
 
 	void DumpProgramState(void);
+
+	virtual void OnShutdown(void);
 };
 
 }

lib/icinga/macroprocessor.cpp

@@ -88,5 +88,7 @@ Dictionary::Ptr MacroProcessor::MergeMacroDicts(const vector<Dictionary::Ptr>& d
 		}
 	}
 
+	result->Seal();
+
 	return result;
 }

lib/icinga/notification.cpp

@@ -70,24 +70,94 @@ Dictionary::Ptr Notification::GetMacros(void) const
 	return Get("macros");
 }
 
-void Notification::SendNotification(NotificationType type)
+String Notification::NotificationTypeToString(NotificationType type)
 {
-	vector<Value> arguments;
+	switch (type) {
-	arguments.push_back(static_cast<Notification::Ptr>(GetSelf()));
+		case NotificationDowntimeStart:
-	arguments.push_back(type);
+			return "DOWNTIMESTART";
+		case NotificationDowntimeEnd:
+			return "DOWNTIMEEND";
+		case NotificationDowntimeRemoved:
+			return "DOWNTIMECANCELLED";
+		case NotificationCustom:
+			return "DOWNTIMECUSTOM";
+		case NotificationProblem:
+			return "PROBLEM";
+		case NotificationRecovery:
+			return "RECOVERY";
+		default:
+			return "UNKNOWN_NOTIFICATION";
+	}
+}
 
-	ScriptTask::Ptr task = MakeMethodTask("notify", arguments);
+void Notification::BeginExecuteNotification(const Notification::Ptr& self, NotificationType type)
+{
 
-	if (!task) {
+	vector<Dictionary::Ptr> macroDicts;
-		Logger::Write(LogWarning, "icinga", "Notification object '" + GetName() + "' doesn't have a 'notify' method.");
 
-		return;
+	Dictionary::Ptr notificationMacros = boost::make_shared<Dictionary>();
+	notificationMacros->Set("NOTIFICATIONTYPE", NotificationTypeToString(type));
+	macroDicts.push_back(notificationMacros);
+
+	Service::Ptr service;
+
+	{
+		ObjectLock olock(self);
+		macroDicts.push_back(self->GetMacros());
+		service = self->GetService();
 	}
 
-	/* We need to keep the task object alive until the completion handler is called. */
+	Host::Ptr host;
-	m_Tasks.insert(task);
+	String service_name;
 
-	task->Start(boost::bind(&Notification::NotificationCompletedHandler, this, _1));
+	{
+		ObjectLock olock(service);
+		macroDicts.push_back(service->GetMacros());
+		service_name = service->GetName();
+		host = service->GetHost();
+	}
+
+	macroDicts.push_back(Service::CalculateDynamicMacros(service));
+
+	{
+		ObjectLock olock(host);
+		macroDicts.push_back(host->GetMacros());
+		macroDicts.push_back(Host::CalculateDynamicMacros(host));
+	}
+
+	IcingaApplication::Ptr app = IcingaApplication::GetInstance();
+
+	{
+		ObjectLock olock(app);
+		macroDicts.push_back(app->GetMacros());
+	}
+
+	macroDicts.push_back(IcingaApplication::CalculateDynamicMacros(app));
+
+	Dictionary::Ptr macros = MacroProcessor::MergeMacroDicts(macroDicts);
+
+	vector<Value> arguments;
+	arguments.push_back(self);
+	arguments.push_back(macros);
+	arguments.push_back(type);
+
+	ScriptTask::Ptr task;
+
+	{
+		ObjectLock olock(self);
+		task = self->MakeMethodTask("notify", arguments);
+
+		if (!task) {
+			Logger::Write(LogWarning, "icinga", "Notification object '" + self->GetName() + "' doesn't have a 'notify' method.");
+
+			return;
+		}
+
+		/* We need to keep the task object alive until the completion handler is called. */
+		self->m_Tasks.insert(task);
+	}
+
+	task->Start(boost::bind(&Notification::NotificationCompletedHandler, self, _1));
 }
 
 void Notification::NotificationCompletedHandler(const ScriptTask::Ptr& task)

lib/icinga/notification.h

@@ -34,7 +34,8 @@ enum NotificationType
 	NotificationDowntimeEnd,
 	NotificationDowntimeRemoved,
 	NotificationCustom,
-	NotificationStateChange
+	NotificationProblem,
+	NotificationRecovery
 };
 
 class Service;
@@ -60,7 +61,9 @@ public:
 	Value GetNotificationCommand(void) const;
 	Dictionary::Ptr GetMacros(void) const;
 
-	void SendNotification(NotificationType type);
+	static void BeginExecuteNotification(const Notification::Ptr& self, NotificationType type);
+
+	static String NotificationTypeToString(NotificationType type);
 
 protected:
 	void OnAttributeChanged(const String& name, const Value& oldValue);

lib/icinga/pluginchecktask.cpp

@@ -32,37 +32,19 @@ void PluginCheckTask::ScriptFunc(const ScriptTask::Ptr& task, const vector<Value
 	if (arguments.size() < 1)
 		BOOST_THROW_EXCEPTION(invalid_argument("Missing argument: Service must be specified."));
 
-	Value vservice = arguments[0];
+	if (arguments.size() < 2)
-	if (!vservice.IsObjectType<Service>())
+		BOOST_THROW_EXCEPTION(invalid_argument("Missing argument: Macros must be specified."));
-		BOOST_THROW_EXCEPTION(invalid_argument("Argument must be a service."));
+
+	Service::Ptr service = arguments[0];
+	Dictionary::Ptr macros = arguments[1];
 
-	vector<Dictionary::Ptr> macroDicts;
 	Value raw_command;
-	Host::Ptr host;
 
 	{
-		Service::Ptr service = vservice;
 		ObjectLock olock(service);
-		macroDicts.push_back(service->GetMacros());
-		macroDicts.push_back(service->CalculateDynamicMacros());
 		raw_command = service->GetCheckCommand();
-		host = service->GetHost();
 	}
 
-	{
-		ObjectLock olock(host);
-		macroDicts.push_back(host->GetMacros());
-		macroDicts.push_back(host->CalculateDynamicMacros());
-	}
-
-	{
-		IcingaApplication::Ptr app = IcingaApplication::GetInstance();
-		ObjectLock olock(app);
-		macroDicts.push_back(app->GetMacros());
-	}
-
-	Dictionary::Ptr macros = MacroProcessor::MergeMacroDicts(macroDicts);
-
 	Value command = MacroProcessor::ResolveMacros(raw_command, macros);
 
 	Process::Ptr process = boost::make_shared<Process>(Process::SplitCommand(command), macros);

lib/icinga/pluginnotificationtask.cpp

@@ -37,49 +37,30 @@ void PluginNotificationTask::ScriptFunc(const ScriptTask::Ptr& task, const vecto
 		BOOST_THROW_EXCEPTION(invalid_argument("Missing argument: Notification target must be specified."));
 
 	if (arguments.size() < 2)
+		BOOST_THROW_EXCEPTION(invalid_argument("Missing argument: Macros must be specified."));
+
+	if (arguments.size() < 3)
 		BOOST_THROW_EXCEPTION(invalid_argument("Missing argument: Notification type must be specified."));
 
-	if (!arguments[0].IsObjectType<Notification>())
+	Notification::Ptr notification = arguments[0];
-		BOOST_THROW_EXCEPTION(invalid_argument("Argument must be a service."));
+	Dictionary::Ptr macros = arguments[1];
+	NotificationType type = static_cast<NotificationType>(static_cast<int>(arguments[2]));
 
-	NotificationType type = static_cast<NotificationType>(static_cast<int>(arguments[1]));
-
-	vector<Dictionary::Ptr> macroDicts;
 	Value raw_command;
-	Service::Ptr service;
-	Host::Ptr host;
 	String service_name;
+	Service::Ptr service;
 
 	{
-		Notification::Ptr notification = arguments[0];
 		ObjectLock olock(notification);
-		macroDicts.push_back(notification->GetMacros());
 		raw_command = notification->GetNotificationCommand();
 		service = notification->GetService();
 	}
 
 	{
 		ObjectLock olock(service);
-		macroDicts.push_back(service->GetMacros());
-		macroDicts.push_back(service->CalculateDynamicMacros());
 		service_name = service->GetName();
-		host = service->GetHost();
 	}
 
-	{
-		ObjectLock olock(host);
-		macroDicts.push_back(host->GetMacros());
-		macroDicts.push_back(host->CalculateDynamicMacros());
-	}
-
-	{
-		IcingaApplication::Ptr app = IcingaApplication::GetInstance();
-		ObjectLock olock(app);
-		macroDicts.push_back(app->GetMacros());
-	}
-
-	Dictionary::Ptr macros = MacroProcessor::MergeMacroDicts(macroDicts);
-
 	Value command = MacroProcessor::ResolveMacros(raw_command, macros);
 
 	Process::Ptr process = boost::make_shared<Process>(Process::SplitCommand(command), macros);

lib/icinga/service-check.cpp

@@ -287,6 +287,7 @@ void Service::ApplyCheckResult(const Dictionary::Ptr& cr)
 	ServiceState old_state = GetState();
 	ServiceStateType old_stateType = GetStateType();
 	bool hardChange = false;
+	bool recovery;
 
 	long attempt = GetCurrentCheckAttempt();
 
@@ -298,6 +299,7 @@ void Service::ApplyCheckResult(const Dictionary::Ptr& cr)
 			SetStateType(StateTypeHard);
 
 		attempt = 1;
+		recovery = true;
 	} else {
 		if (attempt >= GetMaxCheckAttempts()) {
 			SetStateType(StateTypeHard);
@@ -307,6 +309,8 @@ void Service::ApplyCheckResult(const Dictionary::Ptr& cr)
 			SetStateType(StateTypeSoft);
 			attempt++;
 		}
+
+		recovery = false;
 	}
 
 	SetCurrentCheckAttempt(attempt);
@@ -353,19 +357,19 @@ void Service::ApplyCheckResult(const Dictionary::Ptr& cr)
 		Flush();
 
 		if (IsReachable(GetSelf()) && !IsInDowntime() && !IsAcknowledged())
-			RequestNotifications(NotificationStateChange);
+			RequestNotifications(recovery ? NotificationRecovery : NotificationProblem);
 	}
 }
 
 ServiceState Service::StateFromString(const String& state)
 {
-	if (state == "ok")
+	if (state == "OK")
 		return StateOK;
-	else if (state == "warning")
+	else if (state == "WARNING")
 		return StateWarning;
-	else if (state == "critical")
+	else if (state == "CRITICAL")
 		return StateCritical;
-	else if (state == "uncheckable")
+	else if (state == "UNCHECKABLE")
 		return StateUncheckable;
 	else
 		return StateUnknown;
@@ -375,22 +379,22 @@ String Service::StateToString(ServiceState state)
 {
 	switch (state) {
 		case StateOK:
-			return "ok";
+			return "OK";
 		case StateWarning:
-			return "warning";
+			return "WARNING";
 		case StateCritical:
-			return "critical";
+			return "CRITICAL";
 		case StateUncheckable:
-			return "uncheckable";
+			return "UNCHECKABLE";
 		case StateUnknown:
 		default:
-			return "unknown";
+			return "UNKNOWN";
 	}
 }
 
 ServiceStateType Service::StateTypeFromString(const String& type)
 {
-	if (type == "soft")
+	if (type == "SOFT")
 		return StateTypeSoft;
 	else
 		return StateTypeHard;
@@ -399,9 +403,9 @@ ServiceStateType Service::StateTypeFromString(const String& type)
 String Service::StateTypeToString(ServiceStateType type)
 {
 	if (type == StateTypeSoft)
-		return "soft";
+		return "SOFT";
 	else
-		return "hard";
+		return "HARD";
 }
 
 bool Service::IsAllowedChecker(const String& checker) const
@@ -420,10 +424,14 @@ bool Service::IsAllowedChecker(const String& checker) const
 	return false;
 }
 
-void Service::BeginExecuteCheck(const function<void (void)>& callback)
+void Service::BeginExecuteCheck(const Service::Ptr& self, const function<void (void)>& callback)
 {
+	ObjectLock slock(self);
+
 	/* don't run another check if there is one pending */
-	if (!Get("current_task").IsEmpty()) {
+	if (!self->Get("current_task").IsEmpty()) {
+		slock.Unlock();
+
 		/* we need to call the callback anyway */
 		callback();
 
@@ -431,28 +439,59 @@ void Service::BeginExecuteCheck(const function<void (void)>& callback)
 	}
 
 	/* keep track of scheduling info in case the check type doesn't provide its own information */
-	Dictionary::Ptr scheduleInfo = boost::make_shared<Dictionary>();
+	Dictionary::Ptr checkInfo = boost::make_shared<Dictionary>();
-	scheduleInfo->Set("schedule_start", GetNextCheck());
+	checkInfo->Set("schedule_start", self->GetNextCheck());
-	scheduleInfo->Set("execution_start", Utility::GetTime());
+	checkInfo->Set("execution_start", Utility::GetTime());
+
+	vector<Dictionary::Ptr> macroDicts;
+	macroDicts.push_back(self->GetMacros());
+	macroDicts.push_back(Service::CalculateDynamicMacros(self));
+
+	Value raw_command = self->GetCheckCommand();
+
+	Host::Ptr host = self->GetHost();
+
+	slock.Unlock();
+
+	{
+		ObjectLock olock(host);
+		macroDicts.push_back(host->GetMacros());
+		macroDicts.push_back(Host::CalculateDynamicMacros(host));
+	}
+
+	IcingaApplication::Ptr app = IcingaApplication::GetInstance();
+
+	{
+		ObjectLock olock(app);
+		macroDicts.push_back(app->GetMacros());
+	}
+
+	macroDicts.push_back(IcingaApplication::CalculateDynamicMacros(app));
+
+	Dictionary::Ptr macros = MacroProcessor::MergeMacroDicts(macroDicts);
+
+	checkInfo->Set("macros", macros);
 
 	vector<Value> arguments;
-	arguments.push_back(static_cast<Service::Ptr>(GetSelf()));
+	arguments.push_back(self);
+	arguments.push_back(macros);
 
-	ScriptTask::Ptr task = MakeMethodTask("check", arguments);
+	ScriptTask::Ptr task;
-	Set("current_task", task);
 
-	task->Start(boost::bind(&Service::CheckCompletedHandler, this, scheduleInfo, _1, callback));
+	{
+		ObjectLock olock(self);
+		task = self->MakeMethodTask("check", arguments);
+		self->Set("current_task", task);
+	}
+
+	task->Start(boost::bind(&Service::CheckCompletedHandler, self, checkInfo, _1, callback));
 }
 
-void Service::CheckCompletedHandler(const Dictionary::Ptr& scheduleInfo,
+void Service::CheckCompletedHandler(const Dictionary::Ptr& checkInfo,
     const ScriptTask::Ptr& task, const function<void (void)>& callback)
 {
-	ObjectLock olock(this);
+	checkInfo->Set("execution_end", Utility::GetTime());
-
+	checkInfo->Set("schedule_end", Utility::GetTime());
-	Set("current_task", Empty);
-
-	scheduleInfo->Set("execution_end", Utility::GetTime());
-	scheduleInfo->Set("schedule_end", Utility::GetTime());
 
 	Dictionary::Ptr result;
 
@@ -481,32 +520,43 @@ void Service::CheckCompletedHandler(const Dictionary::Ptr& scheduleInfo,
 
 	if (result) {
 		if (!result->Contains("schedule_start"))
-			result->Set("schedule_start", scheduleInfo->Get("schedule_start"));
+			result->Set("schedule_start", checkInfo->Get("schedule_start"));
 
 		if (!result->Contains("schedule_end"))
-			result->Set("schedule_end", scheduleInfo->Get("schedule_end"));
+			result->Set("schedule_end", checkInfo->Get("schedule_end"));
 
 		if (!result->Contains("execution_start"))
-			result->Set("execution_start", scheduleInfo->Get("execution_start"));
+			result->Set("execution_start", checkInfo->Get("execution_start"));
 
 		if (!result->Contains("execution_end"))
-			result->Set("execution_end", scheduleInfo->Get("execution_end"));
+			result->Set("execution_end", checkInfo->Get("execution_end"));
+
+		if (!result->Contains("macros"))
+			result->Set("macros", checkInfo->Get("macros"));
 
 		if (!result->Contains("active"))
 			result->Set("active", 1);
 
-		if (!result->Contains("checker"))
+		if (!result->Contains("checker")) {
-			result->Set("checker", EndpointManager::GetInstance()->GetIdentity());
+			EndpointManager::Ptr em = EndpointManager::GetInstance();
+			ObjectLock olock(em);
 
-		ProcessCheckResult(result);
+			result->Set("checker", em->GetIdentity());
+		}
 	}
 
-	/* figure out when the next check is for this service; the call to
+	{
-	 * ApplyCheckResult() should've already done this but lets do it again
+		ObjectLock olock(this);
-	 * just in case there was no check result. */
+		if (result)
-	UpdateNextCheck();
+			ProcessCheckResult(result);
 
-	olock.Unlock();
+		Set("current_task", Empty);
+
+		/* figure out when the next check is for this service; the call to
+		 * ApplyCheckResult() should've already done this but lets do it again
+		 * just in case there was no check result. */
+		UpdateNextCheck();
+	}
 
 	callback();
 }
@@ -551,3 +601,40 @@ void Service::UpdateStatistics(const Dictionary::Ptr& cr)
 	else
 		CIB::UpdatePassiveChecksStatistics(ts, 1);
 }
+
+double Service::CalculateExecutionTime(const Dictionary::Ptr& cr)
+{
+	ObjectLock olock(cr);
+
+	double execution_start = 0, execution_end = 0;
+
+	if (cr) {
+		ObjectLock olock(cr);
+
+		if (!cr->Contains("execution_start") || !cr->Contains("execution_end"))
+			return 0;
+
+		execution_start = cr->Get("execution_start");
+		execution_end = cr->Get("execution_end");
+	}
+
+	return (execution_end - execution_start);
+}
+
+double Service::CalculateLatency(const Dictionary::Ptr& cr)
+{
+	double schedule_start = 0, schedule_end = 0;
+
+	if (cr) {
+		ObjectLock olock(cr);
+
+		if (!cr->Contains("schedule_start") || !cr->Contains("schedule_end"))
+			return 0;
+
+		schedule_start = cr->Get("schedule_start");
+		schedule_end = cr->Get("schedule_end");
+	}
+
+	return (schedule_end - schedule_start) - CalculateExecutionTime(cr);
+
+}

lib/icinga/service-notification.cpp

@@ -49,7 +49,7 @@ void Service::SendNotifications(NotificationType type)
 		Logger::Write(LogInformation, "icinga", "Service '" + GetName() + "' does not have any notifications.");
 
 	BOOST_FOREACH(const Notification::Ptr& notification, notifications) {
-		notification->SendNotification(type);
+		Notification::BeginExecuteNotification(notification, type);
 	}
 
 	SetLastNotification(Utility::GetTime());

lib/icinga/service.cpp

@@ -76,10 +76,10 @@ String Service::GetDisplayName(void) const
 {
 	String value = Get("display_name");
 
-	if (!value.IsEmpty())
+	if (value.IsEmpty())
-		return value;
+		return GetShortName();
 
-	return GetName();
+	return value;
 }
 
 /**
@@ -330,27 +330,36 @@ set<Service::Ptr> Service::GetParentServices(void) const
 	return parents;
 }
 
-Dictionary::Ptr Service::CalculateDynamicMacros(void) const
+Dictionary::Ptr Service::CalculateDynamicMacros(const Service::Ptr& self)
 {
 	Dictionary::Ptr macros = boost::make_shared<Dictionary>();
 
-	macros->Set("SERVICEDESC", GetShortName());
+	Dictionary::Ptr cr;
-	macros->Set("SERVICEDISPLAYNAME", GetDisplayName());
-	macros->Set("SERVICESTATE", StateToString(GetState()));
-	macros->Set("SERVICESTATEID", GetState());
-	macros->Set("SERVICESTATETYPE", StateTypeToString(GetStateType()));
-	macros->Set("SERVICEATTEMPT", GetCurrentCheckAttempt());
-	macros->Set("MAXSERVICEATTEMPT", GetMaxCheckAttempts());
 
-	Dictionary::Ptr cr = GetLastCheckResult();
+	{
+		ObjectLock olock(self);
+		macros->Set("SERVICEDESC", self->GetShortName());
+		macros->Set("SERVICEDISPLAYNAME", self->GetDisplayName());
+		macros->Set("SERVICESTATE", StateToString(self->GetState()));
+		macros->Set("SERVICESTATEID", self->GetState());
+		macros->Set("SERVICESTATETYPE", StateTypeToString(self->GetStateType()));
+		macros->Set("SERVICEATTEMPT", self->GetCurrentCheckAttempt());
+		macros->Set("MAXSERVICEATTEMPT", self->GetMaxCheckAttempts());
+
+		cr = self->GetLastCheckResult();
+	}
 
 	if (cr) {
+		macros->Set("SERVICELATENCY", Service::CalculateLatency(cr));
+		macros->Set("SERVICEEXECUTIONTIME", Service::CalculateExecutionTime(cr));
+
+		ObjectLock olock(cr);
+
 		macros->Set("SERVICEOUTPUT", cr->Get("output"));
 		macros->Set("SERVICEPERFDATA", cr->Get("performance_data_raw"));
-	} else {
-		macros->Set("SERVICEOUTPUT", "");
-		macros->Set("SERVICEPERFDATA", "");
 	}
 
+	macros->Seal();
+
 	return macros;
 }

lib/icinga/service.h

@@ -106,7 +106,7 @@ public:
 	Dictionary::Ptr GetGroups(void) const;
 	String GetShortName(void) const;
 
-	Dictionary::Ptr CalculateDynamicMacros(void) const;
+	static Dictionary::Ptr CalculateDynamicMacros(const Service::Ptr& self);
 
 	set<Host::Ptr> GetParentHosts(void) const;
 	set<Service::Ptr> GetParentServices(void) const;
@@ -171,9 +171,12 @@ public:
 	void ApplyCheckResult(const Dictionary::Ptr& cr);
 	static void UpdateStatistics(const Dictionary::Ptr& cr);
 
-	void BeginExecuteCheck(const function<void (void)>& callback);
+	static void BeginExecuteCheck(const Service::Ptr& self, const function<void (void)>& callback);
 	void ProcessCheckResult(const Dictionary::Ptr& cr);
 
+	static double CalculateExecutionTime(const Dictionary::Ptr& cr);
+	static double CalculateLatency(const Dictionary::Ptr& cr);
+
 	static ServiceState StateFromString(const String& state);
 	static String StateToString(ServiceState state);
 
@@ -252,7 +255,7 @@ protected:
 	virtual void OnAttributeChanged(const String& name, const Value& oldValue);
 
 private:
-	void CheckCompletedHandler(const Dictionary::Ptr& scheduleInfo,
+	void CheckCompletedHandler(const Dictionary::Ptr& checkInfo,
 	    const ScriptTask::Ptr& task, const function<void (void)>& callback);
 
 	/* Downtimes */

lib/remoting/endpointmanager.cpp

@@ -341,6 +341,8 @@ void EndpointManager::SubscriptionTimerHandler(void)
 		}
 	}
 
+	subscriptions->Seal();
+
 	if (m_Endpoint)
 		m_Endpoint->SetSubscriptions(subscriptions);
 }