Implement include_recursive config directive.

Fixes #5238

doc/2.1-setting-up-icinga-2.md

@@ -99,10 +99,10 @@ the features which have been enabled with `icinga2-enable-feature`. See
 
     /**
      * Although in theory you could define all your objects in this file
-     * the preferred way is to create separate files in the conf.d
+     * the preferred way is to create separate directories and files in the conf.d
      * directory.
      */
-    include "conf.d/*.conf"
+    include_recursive "conf.d" "*.conf"
 
 You can put your own configuration files in the `conf.d` directory. This
 directive makes sure that all of your own configuration files are included.

doc/4.1-configuration-syntax.md

@@ -351,6 +351,22 @@ paths. Additional include search paths can be added using
 
 Wildcards are not permitted when using angle brackets.
 
+### Recursive Includes
+
+The `include_recursive` directive can be used to recursively include all
+files in a directory which match a certain pattern.
+
+Example:
+
+    include_recursive "conf.d" "*.conf"
+    include_recursive "templates"
+
+The first parameter specifies the directory from which files should be
+recursively included.
+
+The file names need to match the pattern given in the second parameter.
+When no pattern is specified the default pattern "*.conf" is used.
+
 ### <a id="library"></a> Library directive
 
 The `library` directive can be used to manually load additional

etc/icinga2/icinga2.conf

@@ -19,8 +19,8 @@ include "features-enabled/*.conf"
 
 /**
  * Although in theory you could define all your objects in this file
- * the preferred way is to create separate files in the conf.d
+ * the preferred way is to create separate directories and files in the conf.d
  * directory.
  */
-include "conf.d/*.conf"
+include_recursive "conf.d" "*.conf"
 

lib/base/unix.h

@@ -38,6 +38,7 @@
 #include <dlfcn.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <dirent.h>
 
 typedef int SOCKET;
 #define INVALID_SOCKET (-1)

lib/base/utility.cpp

@@ -386,7 +386,9 @@ bool Utility::Glob(const String& pathSpec, const boost::function<void (const Str
 		struct stat statbuf;
 
 		if (stat(*gp, &statbuf) < 0)
-			continue;
+			BOOST_THROW_EXCEPTION(posix_error()
+			    << boost::errinfo_api_function("stat")
+			    << boost::errinfo_errno(errno));
 
 		if (!S_ISDIR(statbuf.st_mode) && !S_ISREG(statbuf.st_mode))
 			continue;
@@ -406,6 +408,121 @@ bool Utility::Glob(const String& pathSpec, const boost::function<void (const Str
 #endif /* _WIN32 */
 }
 
+/**
+ * Calls the specified callback for each file in the specified directory
+ * or any of its child directories if the file name matches the specified
+ * pattern.
+ *
+ * @param path The path.
+ * @param pattern The pattern.
+ * @param callback The callback which is invoked for each matching file.
+ * @param type The file type (a combination of GlobFile and GlobDirectory)
+ */
+bool Utility::GlobRecursive(const String& path, const String& pattern, const boost::function<void (const String&)>& callback, int type)
+{
+#ifdef _WIN32
+	HANDLE handle;
+	WIN32_FIND_DATA wfd;
+
+	String pathSpec = path + "/*";
+
+	handle = FindFirstFile(pathSpec.CStr(), &wfd);
+
+	if (handle == INVALID_HANDLE_VALUE) {
+		DWORD errorCode = GetLastError();
+
+		if (errorCode == ERROR_FILE_NOT_FOUND)
+			return false;
+
+		BOOST_THROW_EXCEPTION(win32_error()
+		    << boost::errinfo_api_function("FindFirstFile")
+			<< errinfo_win32_error(errorCode)
+		    << boost::errinfo_file_name(pathSpec));
+	}
+
+	do {
+		String cpath = path + "/" + wfd.cFileName;
+
+		if (wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+			GlobRecursive(cpath, pattern, callback, type);
+
+		if ((wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && !(type & GlobDirectory))
+			continue;
+
+		if (!(wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && !(type & GlobFile))
+			continue;
+
+		if (!Utility::Match(pattern, wfd.cFileName))
+			continue;
+
+		callback(cpath);
+	} while (FindNextFile(handle, &wfd));
+
+	if (!FindClose(handle)) {
+		BOOST_THROW_EXCEPTION(win32_error()
+		    << boost::errinfo_api_function("FindClose")
+		    << errinfo_win32_error(GetLastError()));
+	}
+
+	return true;
+#else /* _WIN32 */
+	DIR *dirp;
+
+	dirp = opendir(path.CStr());
+
+	if (dirp == NULL)
+		BOOST_THROW_EXCEPTION(posix_error()
+		    << boost::errinfo_api_function("opendir")
+		    << boost::errinfo_errno(errno));
+
+	while (dirp) {
+		dirent ent, *pent;
+
+		if (readdir_r(dirp, &ent, &pent) < 0)
+			BOOST_THROW_EXCEPTION(posix_error()
+			    << boost::errinfo_api_function("readdir_r")
+			    << boost::errinfo_errno(errno));
+
+		if (!pent)
+			break;
+
+		if (strcmp(ent.d_name, ".") == 0 || strcmp(ent.d_name, "..") == 0)
+			continue;
+
+		String cpath = path + "/" + ent.d_name;
+
+		struct stat statbuf;
+
+		if (lstat(cpath.CStr(), &statbuf) < 0)
+			BOOST_THROW_EXCEPTION(posix_error()
+			    << boost::errinfo_api_function("lstat")
+			    << boost::errinfo_errno(errno));
+
+		if (S_ISDIR(statbuf.st_mode))
+			GlobRecursive(cpath, pattern, callback, type);
+
+		if (stat(cpath.CStr(), &statbuf) < 0)
+			BOOST_THROW_EXCEPTION(posix_error()
+			    << boost::errinfo_api_function("stat")
+			    << boost::errinfo_errno(errno));
+
+		if (!S_ISDIR(statbuf.st_mode) && !S_ISREG(statbuf.st_mode))
+			continue;
+
+		if (S_ISDIR(statbuf.st_mode) && !(type & GlobDirectory))
+			continue;
+
+		if (!S_ISDIR(statbuf.st_mode) && !(type & GlobFile))
+			continue;
+
+		if (!Utility::Match(pattern, ent.d_name))
+			continue;
+
+		callback(cpath);
+	}
+#endif /* _WIN32 */
+}
+
 #ifndef _WIN32
 void Utility::SetNonBlocking(int fd)
 {

lib/base/utility.h

@@ -76,6 +76,7 @@ public:
 	static String NewUniqueID(void);
 
 	static bool Glob(const String& pathSpec, const boost::function<void (const String&)>& callback, int type = GlobFile | GlobDirectory);
+	static bool GlobRecursive(const String& path, const String& pattern, const boost::function<void (const String&)>& callback, int type = GlobFile | GlobDirectory);
 
 	static void QueueAsyncCallback(const boost::function<void (void)>& callback);
 

lib/config/config_lexer.ll

@@ -212,6 +212,7 @@ name				{ yylval->type = TypeName; return T_TYPE_NAME; }
 object				return T_OBJECT;
 template			return T_TEMPLATE;
 include				return T_INCLUDE;
+include_recursive		return T_INCLUDE_RECURSIVE;
 library				return T_LIBRARY;
 inherits			return T_INHERITS;
 null				return T_NULL;

lib/config/config_parser.yy

@@ -91,6 +91,7 @@ using namespace icinga;
 %token T_OBJECT "object (T_OBJECT)"
 %token T_TEMPLATE "template (T_TEMPLATE)"
 %token T_INCLUDE "include (T_INCLUDE)"
+%token T_INCLUDE_RECURSIVE "include_recursive (T_INCLUDE_RECURSIVE)"
 %token T_LIBRARY "library (T_LIBRARY)"
 %token T_INHERITS "inherits (T_INHERITS)"
 %token T_PARTIAL "partial (T_PARTIAL)"
@@ -153,7 +154,7 @@ statements: /* empty */
 	| statements statement
 	;
 
-statement: object | type | include | library | variable
+statement: object | type | include | include_recursive | library | variable
 	;
 
 include: T_INCLUDE value
@@ -166,6 +167,20 @@ include: T_INCLUDE value
 		context->HandleInclude($2, true, yylloc);
 		free($2);
 	}
+	;
+
+include_recursive: T_INCLUDE_RECURSIVE value
+	{
+		context->HandleIncludeRecursive(*$2, "*.conf", yylloc);
+		delete $2;
+	}
+	| T_INCLUDE_RECURSIVE value value
+	{
+		context->HandleIncludeRecursive(*$2, *$3, yylloc);
+		delete $2;
+		delete $3;
+	}
+	;
 
 library: T_LIBRARY T_STRING
 	{

lib/config/configcompiler.cpp

@@ -38,11 +38,9 @@ std::vector<String> ConfigCompiler::m_IncludeSearchDirs;
  * @param path The path of the configuration file (or another name that
  *	       identifies the source of the configuration text).
  * @param input Input stream for the configuration file.
- * @param includeHandler Handler function for #include directives.
  */
-ConfigCompiler::ConfigCompiler(const String& path, std::istream *input,
+ConfigCompiler::ConfigCompiler(const String& path, std::istream *input)
-    HandleIncludeFunc includeHandler)
+	: m_Path(path), m_Input(input)
-	: m_Path(path), m_Input(input), m_HandleInclude(includeHandler)
 {
 	InitializeScanner();
 }
@@ -89,8 +87,7 @@ String ConfigCompiler::GetPath(void) const
 }
 
 /**
- * Handles an include directive by calling the include handler callback
+ * Handles an include directive.
- * function.
  *
  * @param include The path from the include directive.
  * @param search Whether to search global include dirs.
@@ -105,7 +102,51 @@ void ConfigCompiler::HandleInclude(const String& include, bool search, const Deb
 	else
 		path = Utility::DirName(GetPath()) + "/" + include;
 
-	m_HandleInclude(path, search, debuginfo);
+	String includePath = path;
+
+	if (search) {
+		BOOST_FOREACH(const String& dir, m_IncludeSearchDirs) {
+			String spath = dir + "/" + include;
+
+#ifndef _WIN32
+			struct stat statbuf;
+			if (lstat(spath.CStr(), &statbuf) >= 0) {
+#else /* _WIN32 */
+			struct _stat statbuf;
+			if (_stat(spath.CStr(), &statbuf) >= 0) {
+#endif /* _WIN32 */
+				includePath = spath;
+				break;
+			}
+		}
+	}
+
+	std::vector<ConfigItem::Ptr> items;
+
+	if (!Utility::Glob(includePath, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile) && includePath.FindFirstOf("*?") == String::NPos) {
+		std::ostringstream msgbuf;
+		msgbuf << "Include file '" + include + "' does not exist: " << debuginfo;
+		BOOST_THROW_EXCEPTION(std::invalid_argument(msgbuf.str()));
+	}
+}
+
+/**
+ * Handles recursive includes.
+ *
+ * @param include The directory path.
+ * @param pattern The file pattern.
+ * @param debuginfo Debug information.
+ */
+void ConfigCompiler::HandleIncludeRecursive(const String& include, const String& pattern, const DebugInfo& debuginfo)
+{
+	String path;
+
+	if (include.GetLength() > 0 && include[0] == '/')
+		path = include;
+	else
+		path = Utility::DirName(GetPath()) + "/" + include;
+
+	Utility::GlobRecursive(path, pattern, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile);
 }
 
 /**
@@ -169,47 +210,6 @@ void ConfigCompiler::CompileText(const String& path, const String& text)
 	return CompileStream(path, &stream);
 }
 
-/**
- * Default include handler. Includes the file and returns a list of
- * configuration items.
- *
- * @param include The path from the include directive.
- * @param search Whether to search include dirs.
- * @param debuginfo Debug information.
- */
-void ConfigCompiler::HandleFileInclude(const String& include, bool search,
-    const DebugInfo& debuginfo)
-{
-	String includePath = include;
-
-	if (search) {
-		String path;
-
-		BOOST_FOREACH(const String& dir, m_IncludeSearchDirs) {
-			String path = dir + "/" + include;
-
-#ifndef _WIN32
-			struct stat statbuf;
-			if (lstat(path.CStr(), &statbuf) >= 0) {
-#else /* _WIN32 */
-			struct _stat statbuf;
-			if (_stat(path.CStr(), &statbuf) >= 0) {
-#endif /* _WIN32 */
-				includePath = path;
-				break;
-			}
-		}
-	}
-
-	std::vector<ConfigItem::Ptr> items;
-
-	if (!Utility::Glob(includePath, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile) && includePath.FindFirstOf("*?") == String::NPos) {
-		std::ostringstream msgbuf;
-		msgbuf << "Include file '" + include + "' does not exist: " << debuginfo;
-		BOOST_THROW_EXCEPTION(std::invalid_argument(msgbuf.str()));
-	}
-}
-
 /**
  * Adds a directory to the list of include search dirs.
  *

lib/config/configcompiler.h

@@ -38,10 +38,7 @@ namespace icinga
 class I2_CONFIG_API ConfigCompiler
 {
 public:
-	typedef boost::function<void (const String&, bool, const DebugInfo&)> HandleIncludeFunc;
+	explicit ConfigCompiler(const String& path, std::istream *input);
-
-	explicit ConfigCompiler(const String& path, std::istream *input,
-	    HandleIncludeFunc includeHandler = &ConfigCompiler::HandleFileInclude);
 	virtual ~ConfigCompiler(void);
 
 	void Compile(void);
@@ -54,11 +51,9 @@ public:
 
 	String GetPath(void) const;
 
-	static void HandleFileInclude(const String& include, bool search,
-	    const DebugInfo& debuginfo);
-
 	/* internally used methods */
 	void HandleInclude(const String& include, bool search, const DebugInfo& debuginfo);
+	void HandleIncludeRecursive(const String& include, const String& pattern, const DebugInfo& debuginfo);
 	void HandleLibrary(const String& library);
 
 	size_t ReadInput(char *buffer, size_t max_bytes);
@@ -68,8 +63,6 @@ private:
 	String m_Path;
 	std::istream *m_Input;
 
-	HandleIncludeFunc m_HandleInclude;
-
 	void *m_Scanner;
 
 	static std::vector<String> m_IncludeSearchDirs;