From f8d2172c27a70771e568f778f7da73c3defb1af7 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Fri, 29 Nov 2013 10:39:48 +0100 Subject: [PATCH] Implement include_recursive config directive. Fixes #5238 --- doc/2.1-setting-up-icinga-2.md | 4 +- doc/4.1-configuration-syntax.md | 16 +++++ etc/icinga2/icinga2.conf | 4 +- lib/base/unix.h | 1 + lib/base/utility.cpp | 119 +++++++++++++++++++++++++++++++- lib/base/utility.h | 1 + lib/config/config_lexer.ll | 1 + lib/config/config_parser.yy | 17 ++++- lib/config/configcompiler.cpp | 96 +++++++++++++------------- lib/config/configcompiler.h | 11 +-- 10 files changed, 207 insertions(+), 63 deletions(-) diff --git a/doc/2.1-setting-up-icinga-2.md b/doc/2.1-setting-up-icinga-2.md index ed28a38b5..4fa3390d1 100644 --- a/doc/2.1-setting-up-icinga-2.md +++ b/doc/2.1-setting-up-icinga-2.md @@ -99,10 +99,10 @@ the features which have been enabled with `icinga2-enable-feature`. See /** * Although in theory you could define all your objects in this file - * the preferred way is to create separate files in the conf.d + * the preferred way is to create separate directories and files in the conf.d * directory. */ - include "conf.d/*.conf" + include_recursive "conf.d" "*.conf" You can put your own configuration files in the `conf.d` directory. This directive makes sure that all of your own configuration files are included. diff --git a/doc/4.1-configuration-syntax.md b/doc/4.1-configuration-syntax.md index 29f76052d..27a812a53 100644 --- a/doc/4.1-configuration-syntax.md +++ b/doc/4.1-configuration-syntax.md @@ -351,6 +351,22 @@ paths. Additional include search paths can be added using Wildcards are not permitted when using angle brackets. +### Recursive Includes + +The `include_recursive` directive can be used to recursively include all +files in a directory which match a certain pattern. + +Example: + + include_recursive "conf.d" "*.conf" + include_recursive "templates" + +The first parameter specifies the directory from which files should be +recursively included. + +The file names need to match the pattern given in the second parameter. +When no pattern is specified the default pattern "*.conf" is used. + ### Library directive The `library` directive can be used to manually load additional diff --git a/etc/icinga2/icinga2.conf b/etc/icinga2/icinga2.conf index b46935c43..5ad8450e9 100644 --- a/etc/icinga2/icinga2.conf +++ b/etc/icinga2/icinga2.conf @@ -19,8 +19,8 @@ include "features-enabled/*.conf" /** * Although in theory you could define all your objects in this file - * the preferred way is to create separate files in the conf.d + * the preferred way is to create separate directories and files in the conf.d * directory. */ -include "conf.d/*.conf" +include_recursive "conf.d" "*.conf" diff --git a/lib/base/unix.h b/lib/base/unix.h index 7c4d0ed61..0f6cf9a12 100644 --- a/lib/base/unix.h +++ b/lib/base/unix.h @@ -38,6 +38,7 @@ #include #include #include +#include typedef int SOCKET; #define INVALID_SOCKET (-1) diff --git a/lib/base/utility.cpp b/lib/base/utility.cpp index f6c258cef..7badf9b3b 100644 --- a/lib/base/utility.cpp +++ b/lib/base/utility.cpp @@ -386,7 +386,9 @@ bool Utility::Glob(const String& pathSpec, const boost::function& callback, int type) +{ +#ifdef _WIN32 + HANDLE handle; + WIN32_FIND_DATA wfd; + + String pathSpec = path + "/*"; + + handle = FindFirstFile(pathSpec.CStr(), &wfd); + + if (handle == INVALID_HANDLE_VALUE) { + DWORD errorCode = GetLastError(); + + if (errorCode == ERROR_FILE_NOT_FOUND) + return false; + + BOOST_THROW_EXCEPTION(win32_error() + << boost::errinfo_api_function("FindFirstFile") + << errinfo_win32_error(errorCode) + << boost::errinfo_file_name(pathSpec)); + } + + do { + String cpath = path + "/" + wfd.cFileName; + + if (wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + GlobRecursive(cpath, pattern, callback, type); + + if ((wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && !(type & GlobDirectory)) + continue; + + if (!(wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && !(type & GlobFile)) + continue; + + if (!Utility::Match(pattern, wfd.cFileName)) + continue; + + callback(cpath); + } while (FindNextFile(handle, &wfd)); + + if (!FindClose(handle)) { + BOOST_THROW_EXCEPTION(win32_error() + << boost::errinfo_api_function("FindClose") + << errinfo_win32_error(GetLastError())); + } + + return true; +#else /* _WIN32 */ + DIR *dirp; + + dirp = opendir(path.CStr()); + + if (dirp == NULL) + BOOST_THROW_EXCEPTION(posix_error() + << boost::errinfo_api_function("opendir") + << boost::errinfo_errno(errno)); + + while (dirp) { + dirent ent, *pent; + + if (readdir_r(dirp, &ent, &pent) < 0) + BOOST_THROW_EXCEPTION(posix_error() + << boost::errinfo_api_function("readdir_r") + << boost::errinfo_errno(errno)); + + if (!pent) + break; + + if (strcmp(ent.d_name, ".") == 0 || strcmp(ent.d_name, "..") == 0) + continue; + + String cpath = path + "/" + ent.d_name; + + struct stat statbuf; + + if (lstat(cpath.CStr(), &statbuf) < 0) + BOOST_THROW_EXCEPTION(posix_error() + << boost::errinfo_api_function("lstat") + << boost::errinfo_errno(errno)); + + if (S_ISDIR(statbuf.st_mode)) + GlobRecursive(cpath, pattern, callback, type); + + if (stat(cpath.CStr(), &statbuf) < 0) + BOOST_THROW_EXCEPTION(posix_error() + << boost::errinfo_api_function("stat") + << boost::errinfo_errno(errno)); + + if (!S_ISDIR(statbuf.st_mode) && !S_ISREG(statbuf.st_mode)) + continue; + + if (S_ISDIR(statbuf.st_mode) && !(type & GlobDirectory)) + continue; + + if (!S_ISDIR(statbuf.st_mode) && !(type & GlobFile)) + continue; + + if (!Utility::Match(pattern, ent.d_name)) + continue; + + callback(cpath); + } +#endif /* _WIN32 */ +} + #ifndef _WIN32 void Utility::SetNonBlocking(int fd) { diff --git a/lib/base/utility.h b/lib/base/utility.h index 269b9af96..ba070a6af 100644 --- a/lib/base/utility.h +++ b/lib/base/utility.h @@ -76,6 +76,7 @@ public: static String NewUniqueID(void); static bool Glob(const String& pathSpec, const boost::function& callback, int type = GlobFile | GlobDirectory); + static bool GlobRecursive(const String& path, const String& pattern, const boost::function& callback, int type = GlobFile | GlobDirectory); static void QueueAsyncCallback(const boost::function& callback); diff --git a/lib/config/config_lexer.ll b/lib/config/config_lexer.ll index 4fa767f63..7ea8a11b5 100644 --- a/lib/config/config_lexer.ll +++ b/lib/config/config_lexer.ll @@ -212,6 +212,7 @@ name { yylval->type = TypeName; return T_TYPE_NAME; } object return T_OBJECT; template return T_TEMPLATE; include return T_INCLUDE; +include_recursive return T_INCLUDE_RECURSIVE; library return T_LIBRARY; inherits return T_INHERITS; null return T_NULL; diff --git a/lib/config/config_parser.yy b/lib/config/config_parser.yy index ced75b6a0..09e76b229 100644 --- a/lib/config/config_parser.yy +++ b/lib/config/config_parser.yy @@ -91,6 +91,7 @@ using namespace icinga; %token T_OBJECT "object (T_OBJECT)" %token T_TEMPLATE "template (T_TEMPLATE)" %token T_INCLUDE "include (T_INCLUDE)" +%token T_INCLUDE_RECURSIVE "include_recursive (T_INCLUDE_RECURSIVE)" %token T_LIBRARY "library (T_LIBRARY)" %token T_INHERITS "inherits (T_INHERITS)" %token T_PARTIAL "partial (T_PARTIAL)" @@ -153,7 +154,7 @@ statements: /* empty */ | statements statement ; -statement: object | type | include | library | variable +statement: object | type | include | include_recursive | library | variable ; include: T_INCLUDE value @@ -166,6 +167,20 @@ include: T_INCLUDE value context->HandleInclude($2, true, yylloc); free($2); } + ; + +include_recursive: T_INCLUDE_RECURSIVE value + { + context->HandleIncludeRecursive(*$2, "*.conf", yylloc); + delete $2; + } + | T_INCLUDE_RECURSIVE value value + { + context->HandleIncludeRecursive(*$2, *$3, yylloc); + delete $2; + delete $3; + } + ; library: T_LIBRARY T_STRING { diff --git a/lib/config/configcompiler.cpp b/lib/config/configcompiler.cpp index ea42c78b9..75e489bd7 100644 --- a/lib/config/configcompiler.cpp +++ b/lib/config/configcompiler.cpp @@ -38,11 +38,9 @@ std::vector ConfigCompiler::m_IncludeSearchDirs; * @param path The path of the configuration file (or another name that * identifies the source of the configuration text). * @param input Input stream for the configuration file. - * @param includeHandler Handler function for #include directives. */ -ConfigCompiler::ConfigCompiler(const String& path, std::istream *input, - HandleIncludeFunc includeHandler) - : m_Path(path), m_Input(input), m_HandleInclude(includeHandler) +ConfigCompiler::ConfigCompiler(const String& path, std::istream *input) + : m_Path(path), m_Input(input) { InitializeScanner(); } @@ -89,8 +87,7 @@ String ConfigCompiler::GetPath(void) const } /** - * Handles an include directive by calling the include handler callback - * function. + * Handles an include directive. * * @param include The path from the include directive. * @param search Whether to search global include dirs. @@ -105,7 +102,51 @@ void ConfigCompiler::HandleInclude(const String& include, bool search, const Deb else path = Utility::DirName(GetPath()) + "/" + include; - m_HandleInclude(path, search, debuginfo); + String includePath = path; + + if (search) { + BOOST_FOREACH(const String& dir, m_IncludeSearchDirs) { + String spath = dir + "/" + include; + +#ifndef _WIN32 + struct stat statbuf; + if (lstat(spath.CStr(), &statbuf) >= 0) { +#else /* _WIN32 */ + struct _stat statbuf; + if (_stat(spath.CStr(), &statbuf) >= 0) { +#endif /* _WIN32 */ + includePath = spath; + break; + } + } + } + + std::vector items; + + if (!Utility::Glob(includePath, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile) && includePath.FindFirstOf("*?") == String::NPos) { + std::ostringstream msgbuf; + msgbuf << "Include file '" + include + "' does not exist: " << debuginfo; + BOOST_THROW_EXCEPTION(std::invalid_argument(msgbuf.str())); + } +} + +/** + * Handles recursive includes. + * + * @param include The directory path. + * @param pattern The file pattern. + * @param debuginfo Debug information. + */ +void ConfigCompiler::HandleIncludeRecursive(const String& include, const String& pattern, const DebugInfo& debuginfo) +{ + String path; + + if (include.GetLength() > 0 && include[0] == '/') + path = include; + else + path = Utility::DirName(GetPath()) + "/" + include; + + Utility::GlobRecursive(path, pattern, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile); } /** @@ -169,47 +210,6 @@ void ConfigCompiler::CompileText(const String& path, const String& text) return CompileStream(path, &stream); } -/** - * Default include handler. Includes the file and returns a list of - * configuration items. - * - * @param include The path from the include directive. - * @param search Whether to search include dirs. - * @param debuginfo Debug information. - */ -void ConfigCompiler::HandleFileInclude(const String& include, bool search, - const DebugInfo& debuginfo) -{ - String includePath = include; - - if (search) { - String path; - - BOOST_FOREACH(const String& dir, m_IncludeSearchDirs) { - String path = dir + "/" + include; - -#ifndef _WIN32 - struct stat statbuf; - if (lstat(path.CStr(), &statbuf) >= 0) { -#else /* _WIN32 */ - struct _stat statbuf; - if (_stat(path.CStr(), &statbuf) >= 0) { -#endif /* _WIN32 */ - includePath = path; - break; - } - } - } - - std::vector items; - - if (!Utility::Glob(includePath, boost::bind(&ConfigCompiler::CompileFile, _1), GlobFile) && includePath.FindFirstOf("*?") == String::NPos) { - std::ostringstream msgbuf; - msgbuf << "Include file '" + include + "' does not exist: " << debuginfo; - BOOST_THROW_EXCEPTION(std::invalid_argument(msgbuf.str())); - } -} - /** * Adds a directory to the list of include search dirs. * diff --git a/lib/config/configcompiler.h b/lib/config/configcompiler.h index d888ad5ec..c38fd4f96 100644 --- a/lib/config/configcompiler.h +++ b/lib/config/configcompiler.h @@ -38,10 +38,7 @@ namespace icinga class I2_CONFIG_API ConfigCompiler { public: - typedef boost::function HandleIncludeFunc; - - explicit ConfigCompiler(const String& path, std::istream *input, - HandleIncludeFunc includeHandler = &ConfigCompiler::HandleFileInclude); + explicit ConfigCompiler(const String& path, std::istream *input); virtual ~ConfigCompiler(void); void Compile(void); @@ -54,11 +51,9 @@ public: String GetPath(void) const; - static void HandleFileInclude(const String& include, bool search, - const DebugInfo& debuginfo); - /* internally used methods */ void HandleInclude(const String& include, bool search, const DebugInfo& debuginfo); + void HandleIncludeRecursive(const String& include, const String& pattern, const DebugInfo& debuginfo); void HandleLibrary(const String& library); size_t ReadInput(char *buffer, size_t max_bytes); @@ -68,8 +63,6 @@ private: String m_Path; std::istream *m_Input; - HandleIncludeFunc m_HandleInclude; - void *m_Scanner; static std::vector m_IncludeSearchDirs;