Improve error reporting

fixes #8136
refs #6070

debian/icinga2-common.dirs

@@ -5,6 +5,7 @@ var/lib/icinga2/api/repository
 var/lib/icinga2/api/zones
 var/log/icinga2
 var/log/icinga2/compat/archives
+var/log/icinga2/crash
 var/spool/icinga2
 var/spool/icinga2/perfdata
 var/spool/icinga2/tmp

debian/icinga2-common.postinst

@@ -37,6 +37,7 @@ case "$1" in
         setperm nagios nagios 0750 /etc/icinga2
 
         setperm nagios adm 2751 /var/log/icinga2
+        setperm nagios adm 2750 /var/log/icinga2/crash
         setperm nagios adm 2751 /var/log/icinga2/compat
         setperm nagios adm 2755 /var/log/icinga2/compat/archives
 

icinga-app/icinga.cpp

@@ -312,7 +312,7 @@ int Main(void)
 			if (vm.count("version")) {
 				std::cout << std::endl;
 
-				Application::DisplayInfoMessage(true);
+				Application::DisplayInfoMessage(std::cout, true);
 
 				return EXIT_SUCCESS;
 			}

icinga2.spec

@@ -462,6 +462,7 @@ exit 0
 
 %attr(0750,%{icinga_user},%{icingacmd_group}) %{_localstatedir}/cache/%{name}
 %attr(0750,%{icinga_user},%{icingacmd_group}) %dir %{_localstatedir}/log/%{name}
+%attr(0750,%{icinga_user},%{icinga_group}) %dir %{_localstatedir}/log/%{name}/crash
 %attr(0750,%{icinga_user},%{icingacmd_group}) %dir %{_localstatedir}/log/%{name}/compat
 %attr(0750,%{icinga_user},%{icingacmd_group}) %dir %{_localstatedir}/log/%{name}/compat/archives
 %attr(0750,%{icinga_user},%{icinga_group}) %{_localstatedir}/lib/%{name}

lib/base/CMakeLists.txt

@@ -71,6 +71,7 @@ set_target_properties (
 )
 
 install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/cache/icinga2\")")
+install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/log/icinga2/crash\")")
 
 install(
   TARGETS base

lib/base/application.cpp

@@ -29,14 +29,15 @@
 #include "base/convert.hpp"
 #include "base/scriptglobal.hpp"
 #include "base/process.hpp"
-#include <sstream>
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/foreach.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/exception/errinfo_api_function.hpp>
 #include <boost/exception/errinfo_errno.hpp>
 #include <boost/exception/errinfo_file_name.hpp>
+#include <sstream>
 #include <iostream>
+#include <fstream>
 
 using namespace icinga;
 
@@ -474,35 +475,35 @@ String Application::GetExePath(const String& argv0)
 /**
  * Display version and path information.
  */
-void Application::DisplayInfoMessage(bool skipVersion)
+void Application::DisplayInfoMessage(std::ostream& os, bool skipVersion)
 {
-	std::cerr << "Application information:" << std::endl;
+	os << "Application information:" << "\n";
 
 	if (!skipVersion)
-		std::cerr << "  Application version: " << GetVersion() << std::endl;
+		os << "  Application version: " << GetVersion() << "\n";
 
-	std::cerr << "  Installation root: " << GetPrefixDir() << std::endl
+	os << "  Installation root: " << GetPrefixDir() << "\n"
-		  << "  Sysconf directory: " << GetSysconfDir() << std::endl
+	   << "  Sysconf directory: " << GetSysconfDir() << "\n"
-		  << "  Run directory: " << GetRunDir() << std::endl
+	   << "  Run directory: " << GetRunDir() << "\n"
-		  << "  Local state directory: " << GetLocalStateDir() << std::endl
+	   << "  Local state directory: " << GetLocalStateDir() << "\n"
-		  << "  Package data directory: " << GetPkgDataDir() << std::endl
+	   << "  Package data directory: " << GetPkgDataDir() << "\n"
-		  << "  State path: " << GetStatePath() << std::endl
+	   << "  State path: " << GetStatePath() << "\n"
-		  << "  Objects path: " << GetObjectsPath() << std::endl
+	   << "  Objects path: " << GetObjectsPath() << "\n"
-		  << "  Vars path: " << GetVarsPath() << std::endl
+	   << "  Vars path: " << GetVarsPath() << "\n"
-		  << "  PID path: " << GetPidPath() << std::endl
+	   << "  PID path: " << GetPidPath() << "\n"
-		  << "  Application type: " << GetApplicationType() << std::endl;
+	   << "  Application type: " << GetApplicationType() << "\n";
 }
 
 /**
  * Displays a message that tells users what to do when they encounter a bug.
  */
-void Application::DisplayBugMessage(void)
+void Application::DisplayBugMessage(std::ostream& os)
 {
-	std::cerr << "***" << std::endl
+	os << "***" << "\n"
-		  << "* This would indicate a runtime problem or configuration error. If you believe this is a bug in Icinga 2" << std::endl
+	   << "* This would indicate a runtime problem or configuration error. If you believe this is a bug in Icinga 2" << "\n"
-		  << "* please submit a bug report at https://dev.icinga.org/ and include this stack trace as well as any other" << std::endl
+	   << "* please submit a bug report at https://dev.icinga.org/ and include this stack trace as well as any other" << "\n"
-		  << "* information that might be useful in order to reproduce this problem." << std::endl
+	   << "* information that might be useful in order to reproduce this problem." << "\n"
-		  << "***" << std::endl;
+	   << "***" << "\n";
 }
 
 #ifndef _WIN32
@@ -538,6 +539,11 @@ void Application::SigUsr1Handler(int)
 	RequestReopenLogs();
 }
 
+String Application::GetCrashReportFilename(void)
+{
+	return GetLocalStateDir() + "/log/icinga2/crash/report." + Convert::ToString(Utility::GetTime());
+}
+
 /**
  * Signal handler for SIGABRT. Helps with debugging ASSERT()s.
  *
@@ -556,13 +562,22 @@ void Application::SigAbrtHandler(int)
 		  << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << std::endl
 		  << std::endl;
 
-	DisplayInfoMessage();
+	String fname = GetCrashReportFilename();
+	std::ofstream ofs;
+	ofs.open(fname.CStr());
+
+	Log(LogCritical, "Application")
+	    << "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'" << "\n";
+
+	DisplayInfoMessage(ofs);
 
 	StackTrace trace;
-	std::cerr << "Stacktrace:" << std::endl;
+	ofs << "Stacktrace:" << "\n";
-	trace.Print(std::cerr, 1);
+	trace.Print(ofs, 1);
 
-	DisplayBugMessage();
+	DisplayBugMessage(ofs);
+
+	ofs.close();
 }
 #else /* _WIN32 */
 /**
@@ -601,21 +616,32 @@ void Application::ExceptionHandler(void)
 	sigaction(SIGABRT, &sa, NULL);
 #endif /* _WIN32 */
 
-	std::cerr << "Caught unhandled exception." << std::endl
+	String fname = GetCrashReportFilename();
-		  << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << std::endl
+	std::ofstream ofs;
-		  << std::endl;
+	ofs.open(fname.CStr());
 
-	DisplayInfoMessage();
+	ofs << "Caught unhandled exception." << "\n"
+	    << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
+	    << "\n";
+
+	DisplayInfoMessage(ofs);
 
 	try {
 		RethrowUncaughtException();
 	} catch (const std::exception& ex) {
-		std::cerr << std::endl
+		Log(LogCritical, "Application")
-			  << DiagnosticInformation(ex)
+		    << DiagnosticInformation(ex, false) << "\n"
-			  << std::endl;
+		    << "\n"
+		    << "Additional information is available in '" << fname << "'" << "\n";
+
+		ofs << "\n"
+		    << DiagnosticInformation(ex)
+		    << "\n";
 	}
 
-	DisplayBugMessage();
+	DisplayBugMessage(ofs);
+
+	ofs.close();
 
 	abort();
 }
@@ -628,17 +654,24 @@ LONG CALLBACK Application::SEHUnhandledExceptionFilter(PEXCEPTION_POINTERS exi)
 
 	l_InExceptionHandler = true;
 
-	DisplayInfoMessage();
+	String fname = GetCrashReportFilename();
+	std::ofstream ofs;
+	ofs.open(fname.CStr());
 
-	std::cerr << "Caught unhandled SEH exception." << std::endl
+	Log(LogCritical, "Application")
-		  << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << std::endl
+	    << "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'";
-		  << std::endl;
+
+	DisplayInfoMessage(ofs);
+
+	ofs << "Caught unhandled SEH exception." << "\n"
+	    << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
+	    << "\n";
 
 	StackTrace trace(exi);
-	std::cerr << "Stacktrace:" << std::endl;
+	ofs << "Stacktrace:" << "\n";
-	trace.Print(std::cerr, 1);
+	trace.Print(ofs, 1);
 
-	DisplayBugMessage();
+	DisplayBugMessage(ofs);
 
 	return EXCEPTION_CONTINUE_SEARCH;
 }

lib/base/application.hpp

@@ -25,6 +25,7 @@
 #include "base/threadpool.hpp"
 #include "base/utility.hpp"
 #include "base/logger.hpp"
+#include <ostream>
 
 namespace icinga
 {
@@ -133,7 +134,7 @@ public:
 	static double GetStartTime(void);
 	static void SetStartTime(double ts);
 
-	static void DisplayInfoMessage(bool skipVersion = false);
+	static void DisplayInfoMessage(std::ostream& os, bool skipVersion = false);
 
 protected:
 	virtual void OnConfigLoaded(void);
@@ -169,11 +170,13 @@ private:
 	static LONG WINAPI SEHUnhandledExceptionFilter(PEXCEPTION_POINTERS exi);
 #endif /* _WIN32 */
 
-	static void DisplayBugMessage(void);
+	static void DisplayBugMessage(std::ostream& os);
 
 	static void SigAbrtHandler(int signum);
 	static void SigUsr1Handler(int signum);
 	static void ExceptionHandler(void);
+
+	static String GetCrashReportFilename(void);
 };
 
 }

lib/base/exception.cpp

@@ -130,7 +130,59 @@ void icinga::SetLastExceptionContext(const ContextTrace& context)
 	l_LastExceptionContext.reset(new ContextTrace(context));
 }
 
-String icinga::DiagnosticInformation(boost::exception_ptr eptr)
+String icinga::DiagnosticInformation(const std::exception& ex, bool verbose, StackTrace *stack, ContextTrace *context)
+{
+	std::ostringstream result;
+
+	const user_error *uex = dynamic_cast<const user_error *>(&ex);
+
+	String message = ex.what();
+
+	if (verbose)
+		result << boost::diagnostic_information(ex);
+	else
+		result << "Error: " << message;
+
+	const ScriptError *dex = dynamic_cast<const ScriptError *>(&ex);
+
+	if (dex && !dex->GetDebugInfo().Path.IsEmpty()) {
+		result << "\nLocation:\n";
+		ShowCodeFragment(result, dex->GetDebugInfo());
+	}
+
+	const posix_error *pex = dynamic_cast<const posix_error *>(&ex);
+
+	if (!uex && verbose) {
+		const StackTrace *st = boost::get_error_info<StackTraceErrorInfo>(ex);
+
+		if (st) {
+			result << *st;
+		} else {
+			result << std::endl;
+
+			if (!stack)
+				stack = GetLastExceptionStack();
+
+			if (stack)
+				result << *stack;
+
+		}
+
+		if (boost::get_error_info<ContextTraceErrorInfo>(ex) == NULL) {
+			result << std::endl;
+
+			if (!context)
+				context = GetLastExceptionContext();
+
+			if (context)
+				result << *context;
+		}
+	}
+
+	return result.str();
+}
+
+String icinga::DiagnosticInformation(boost::exception_ptr eptr, bool verbose)
 {
 	StackTrace *pt = GetLastExceptionStack();
 	StackTrace stack;
@@ -147,7 +199,7 @@ String icinga::DiagnosticInformation(boost::exception_ptr eptr)
 	try {
 		boost::rethrow_exception(eptr);
 	} catch (const std::exception& ex) {
-		return DiagnosticInformation(ex, pt ? &stack : NULL, pc ? &context : NULL);
+		return DiagnosticInformation(ex, verbose, pt ? &stack : NULL, pc ? &context : NULL);
 	}
 
 	return boost::diagnostic_information(eptr);
@@ -174,3 +226,43 @@ DebugInfo ScriptError::GetDebugInfo(void) const
 	return m_DebugInfo;
 }
 
+posix_error::posix_error(void)
+	: m_Message(NULL)
+{ }
+
+posix_error::~posix_error(void) throw()
+{
+	free(m_Message);
+}
+
+const char *posix_error::what(void) const throw()
+{
+	if (!m_Message) {
+		std::ostringstream msgbuf;
+
+		const char * const *func = boost::get_error_info<boost::errinfo_api_function>(*this);
+
+		if (func)
+			msgbuf << "Function call '" << *func << "'";
+		else
+			msgbuf << "Function call";
+
+		const std::string *fname = boost::get_error_info<boost::errinfo_file_name>(*this);
+
+		if (fname)
+			msgbuf << " for file '" << *fname << "'";
+
+		msgbuf << " failed";
+
+		const int *errnum = boost::get_error_info<boost::errinfo_errno>(*this);
+
+		if (errnum)
+			msgbuf << " with error code " << *errnum << ", '" << strerror(*errnum) << "'";
+
+		String str = msgbuf.str();
+		m_Message = strdup(str.CStr());
+	}
+
+	return m_Message;
+}
+

lib/base/exception.hpp

@@ -71,58 +71,34 @@ I2_BASE_API void SetLastExceptionContext(const ContextTrace& context);
 I2_BASE_API void RethrowUncaughtException(void);
 
 typedef boost::error_info<StackTrace, StackTrace> StackTraceErrorInfo;
-typedef boost::error_info<ContextTrace, ContextTrace> ContextTraceErrorInfo;
 
-template<typename T>
+inline std::string to_string(const StackTraceErrorInfo& e)
-String DiagnosticInformation(const T& ex, StackTrace *stack = NULL, ContextTrace *context = NULL)
 {
-	std::ostringstream result;
+	return "";
-
-	const user_error *uex = dynamic_cast<const user_error *>(&ex);
-
-	String message = ex.what();
-
-	if (!uex || message.IsEmpty())
-		result << boost::diagnostic_information(ex);
-	else
-		result << "Error: " << message;
-
-	const ScriptError *dex = dynamic_cast<const ScriptError *>(&ex);
-
-	if (dex && !dex->GetDebugInfo().Path.IsEmpty()) {
-		result << "\nLocation:\n";
-		ShowCodeFragment(result, dex->GetDebugInfo());
-	}
-
-	if (!uex) {
-		if (boost::get_error_info<StackTraceErrorInfo>(ex) == NULL) {
-			result << std::endl;
-
-			if (!stack)
-				stack = GetLastExceptionStack();
-			
-			if (stack)
-				result << *stack;
-				
-		}
-
-		if (boost::get_error_info<ContextTraceErrorInfo>(ex) == NULL) {
-			result << std::endl;
-
-			if (!context)
-				context = GetLastExceptionContext();
-
-			if (context)
-				result << *context;
-		}
-	}
-
-	return result.str();
 }
 
-I2_BASE_API String DiagnosticInformation(boost::exception_ptr eptr);
+typedef boost::error_info<ContextTrace, ContextTrace> ContextTraceErrorInfo;
 
-class I2_BASE_API posix_error : virtual public std::exception, virtual public boost::exception { };
+inline std::string to_string(const ContextTraceErrorInfo& e)
+{
+	std::ostringstream msgbuf;
+	msgbuf << "[Context] = " << e.value();
+	return msgbuf.str();
+}
+
+I2_BASE_API String DiagnosticInformation(const std::exception& ex, bool verbose = true, StackTrace *stack = NULL, ContextTrace *context = NULL);
+I2_BASE_API String DiagnosticInformation(boost::exception_ptr eptr, bool verbose = true);
+
+class I2_BASE_API posix_error : virtual public std::exception, virtual public boost::exception {
+public:
+	posix_error(void);
+	virtual ~posix_error(void) throw();
+
+	virtual const char *what(void) const throw();
+
+private:
+	mutable char *m_Message;
+};
 
 #ifdef _WIN32
 class I2_BASE_API win32_error : virtual public std::exception, virtual public boost::exception { };
@@ -132,7 +108,7 @@ typedef boost::error_info<struct errinfo_win32_error_, int> errinfo_win32_error;
 
 inline std::string to_string(const errinfo_win32_error& e)
 {
-	return Utility::FormatErrorNumber(e.value());
+	return "[errinfo_win32_error] = " + Utility::FormatErrorNumber(e.value()) + "\n";
 }
 #endif /* _WIN32 */
 
@@ -141,7 +117,7 @@ typedef boost::error_info<struct errinfo_getaddrinfo_error_, int> errinfo_getadd
 
 inline std::string to_string(const errinfo_getaddrinfo_error& e)
 {
-	return gai_strerror(e.value());
+	return "[errinfo_getaddrinfo_error] = " + String(gai_strerror(e.value())) + "\n";
 }
 
 struct errinfo_message_;

lib/base/tlsutility.hpp

@@ -68,7 +68,7 @@ inline std::string to_string(const errinfo_openssl_error& e)
 		message = "Unknown error.";
 
 	tmp << code << ", \"" << message << "\"";
-	return tmp.str();
+	return "[errinfo_openssl_error]" + tmp.str() + "\n";
 }
 
 }

lib/config/configcompiler.cpp

@@ -26,8 +26,6 @@
 #include <fstream>
 #include <boost/foreach.hpp>
 
-using std::ifstream;
-
 using namespace icinga;
 
 std::vector<String> ConfigCompiler::m_IncludeSearchDirs;