Implement support for attaching GDB to the Icinga process on crash

fixes #9866

doc/19-language-reference.md

@@ -375,6 +375,7 @@ EnableHostChecks    |**Read-write.** Whether active host checks are globally ena
 EnableServiceChecks |**Read-write.** Whether active service checks are globally enabled. Defaults to true.
 EnablePerfdata      |**Read-write.** Whether performance data processing is globally enabled. Defaults to true.
 UseVfork            |**Read-write.** Whether to use vfork(). Only available on *NIX. Defaults to true.
+AttachDebugger      |**Read-write.** Whether to attach a debugger when Icinga 2 crashes. Defaults to false.
 RunAsUser	        |**Read-write.** Defines the user the Icinga 2 daemon is running as. Used in the `init.conf` configuration file.
 RunAsGroup	        |**Read-write.** Defines the group the Icinga 2 daemon is running as. Used in the `init.conf` configuration file.
 

icinga-app/icinga.cpp

@@ -173,6 +173,8 @@ int Main(void)
 		ScriptGlobal::Set("UseVfork", true);
 #endif /* __APPLE__ */
 
+	ScriptGlobal::Set("AttachDebugger", false);
+
 	LogSeverity logLevel = Logger::GetConsoleLogSeverity();
 	Logger::SetConsoleLogSeverity(LogWarning);
 

lib/base/application.cpp

@@ -626,9 +626,9 @@ String Application::GetCrashReportFilename(void)
 }
 
 
-#ifndef _WIN32
+void Application::AttachDebugger(const String& filename, bool interactive)
-void Application::GetDebuggerBacktrace(const String& filename)
 {
+#ifndef _WIN32
 #ifdef __linux__
 	prctl(PR_SET_DUMPABLE, 1);
 #endif /* __linux __ */
@@ -644,42 +644,58 @@ void Application::GetDebuggerBacktrace(const String& filename)
 	}
 
 	if (pid == 0) {
-		int fd = open(filename.CStr(), O_CREAT | O_RDWR | O_APPEND, 0600);
+		if (!interactive) {
+			int fd = open(filename.CStr(), O_CREAT | O_RDWR | O_APPEND, 0600);
 
-		if (fd < 0) {
+			if (fd < 0) {
-			BOOST_THROW_EXCEPTION(posix_error()
+				BOOST_THROW_EXCEPTION(posix_error()
-			    << boost::errinfo_api_function("open")
+				    << boost::errinfo_api_function("open")
-			    << boost::errinfo_errno(errno)
+				    << boost::errinfo_errno(errno)
-			    << boost::errinfo_file_name(filename));
+				    << boost::errinfo_file_name(filename));
+			}
+
+			if (fd != 1) {
+				/* redirect stdout to the file */
+				dup2(fd, 1);
+				close(fd);
+			}
+
+			/* redirect stderr to stdout */
+			if (fd != 2)
+				close(2);
+
+			dup2(1, 2);
 		}
 
-		if (fd != 1) {
+		char **argv;
-			/* redirect stdout to the file */
-			dup2(fd, 1);
-			close(fd);
-		}
-
-		/* redirect stderr to stdout */
-		if (fd != 2)
-			close(2);
-
-		dup2(1, 2);
-
 		char *my_pid_str = strdup(my_pid.CStr());
-		const char *argv[] = {
+
-			"gdb",
+		if (interactive) {
-			"--batch",
+			const char *uargv[] = {
-			"-p",
+				"gdb",
-			my_pid_str,
+				"-p",
-			"-ex",
+				my_pid_str,
-			"thread apply all bt full",
+				NULL
-			"-ex",
+			};
-			"detach",
+			argv = const_cast<char **>(uargv);
-			"-ex",
+		} else {
-			"quit",
+			const char *uargv[] = {
-			NULL
+				"gdb",
-		};
+				"--batch",
-		(void)execvp(argv[0], const_cast<char **>(argv));
+				"-p",
+				my_pid_str,
+				"-ex",
+				"thread apply all bt full",
+				"-ex",
+				"detach",
+				"-ex",
+				"quit",
+				NULL
+			};
+			argv = const_cast<char **>(uargv);
+		}
+
+		(void)execvp(argv[0], argv);
 		perror("Failed to launch GDB");
 		free(my_pid_str);
 		_exit(0);
@@ -695,6 +711,9 @@ void Application::GetDebuggerBacktrace(const String& filename)
 #ifdef __linux__
 	prctl(PR_SET_DUMPABLE, 0);
 #endif /* __linux __ */
+#else /* _WIN32 */
+	DebugBreak();
+#endif /* _WIN32 */
 }
 
 /**
@@ -750,30 +769,33 @@ void Application::SigAbrtHandler(int)
 	String fname = GetCrashReportFilename();
 	Utility::MkDir(Utility::DirName(fname), 0750);
 
-	std::ofstream ofs;
+	bool interactive_debugger = Convert::ToBool(ScriptGlobal::Get("AttachDebugger"));
-	ofs.open(fname.CStr());
 
-	Log(LogCritical, "Application")
+	if (!interactive_debugger) {
-	    << "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'" << "\n";
+		std::ofstream ofs;
+		ofs.open(fname.CStr());
 
-	DisplayInfoMessage(ofs);
+		Log(LogCritical, "Application")
+		    << "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'" << "\n";
 
-	StackTrace trace;
+		DisplayInfoMessage(ofs);
-	ofs << "Stacktrace:" << "\n";
-	trace.Print(ofs, 1);
 
-	DisplayBugMessage(ofs);
+		StackTrace trace;
+		ofs << "Stacktrace:" << "\n";
+		trace.Print(ofs, 1);
 
-#ifndef _WIN32
+		DisplayBugMessage(ofs);
-	ofs << "\n";
-	ofs.close();
 
-	GetDebuggerBacktrace(fname);
+		ofs << "\n";
-#else /* _WIN32 */
+		ofs.close();
-	ofs.close();
+	} else {
-#endif /* _WIN32 */
+		Log(LogCritical, "Application", "Icinga 2 has terminated unexpeectedly. Attaching debugger...");
+	}
+
+	AttachDebugger(fname, interactive_debugger);
 }
-#else /* _WIN32 */
+
+#ifdef _WIN32
 /**
  * Console control handler. Prepares the application for cleanly
  * shutting down during the next execution of the event loop.
@@ -813,35 +835,37 @@ void Application::ExceptionHandler(void)
 	String fname = GetCrashReportFilename();
 	Utility::MkDir(Utility::DirName(fname), 0750);
 
-	std::ofstream ofs;
+	bool interactive_debugger = Convert::ToBool(ScriptGlobal::Get("AttachDebugger"));
-	ofs.open(fname.CStr());
 
-	ofs << "Caught unhandled exception." << "\n"
+	if (interactive_debugger) {
-	    << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
+		std::ofstream ofs;
-	    << "\n";
+		ofs.open(fname.CStr());
 
-	DisplayInfoMessage(ofs);
+		ofs << "Caught unhandled exception." << "\n"
-
+		    << "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
-	try {
-		RethrowUncaughtException();
-	} catch (const std::exception& ex) {
-		Log(LogCritical, "Application")
-		    << DiagnosticInformation(ex, false) << "\n"
-		    << "\n"
-		    << "Additional information is available in '" << fname << "'" << "\n";
-
-		ofs << "\n"
-		    << DiagnosticInformation(ex)
 		    << "\n";
+
+		DisplayInfoMessage(ofs);
+
+		try {
+			RethrowUncaughtException();
+		} catch (const std::exception& ex) {
+			Log(LogCritical, "Application")
+			    << DiagnosticInformation(ex, false) << "\n"
+			    << "\n"
+			    << "Additional information is available in '" << fname << "'" << "\n";
+
+			ofs << "\n"
+			    << DiagnosticInformation(ex)
+			    << "\n";
+		}
+
+		DisplayBugMessage(ofs);
+
+		ofs.close();
 	}
 
-	DisplayBugMessage(ofs);
+	AttachDebugger(fname, interactive_debugger);
-
-	ofs.close();
-
-#ifndef _WIN32
-	GetDebuggerBacktrace(fname);
-#endif /* _WIN32 */
 
 	abort();
 }

lib/base/application.hpp

@@ -175,9 +175,7 @@ private:
 
 	static String GetCrashReportFilename(void);
 
-#ifndef _WIN32
+	static void AttachDebugger(const String& filename, bool interactive);
-	static void GetDebuggerBacktrace(const String& filename);
-#endif /* _WIN32 */
 };
 
 }