diff --git a/application/clicommands/HealthCommand.php b/application/clicommands/HealthCommand.php new file mode 100644 index 00000000..7be24ea9 --- /dev/null +++ b/application/clicommands/HealthCommand.php @@ -0,0 +1,62 @@ + Run only a specific set of checks + * valid names: config, sync, import, job + * --db Use a specific Icinga Web DB resource + */ + public function checkAction() + { + $health = new Health(); + if ($name = $this->params->get('db')) { + $health->setDbResourceName($name); + } + + if ($name = $this->params->get('check')) { + $check = $health->getCheck($name); + echo $check->getOutput(); + + exit($check->getState()->getNumeric()); + } else { + $state = new PluginState('OK'); + $checks = $health->getAllChecks(); + + $output = []; + foreach ($checks as $check) { + $state->raise($check->getState()); + $output[] = $check->getOutput(); + } + + if ($state === 0) { + echo "Icinga Director: everything is fine\n\n"; + } else { + echo "Icinga Director: there are problems\n\n"; + } + echo implode("\n", $output); + exit($state->getNumeric()); + } + } +} diff --git a/doc/60-CLI.md b/doc/60-CLI.md index 61b9c14b..2242fe48 100644 --- a/doc/60-CLI.md +++ b/doc/60-CLI.md @@ -297,6 +297,51 @@ a good reason. The CLI allows you to issue operations that are not allowed in th web frontend. Do not use this unless you really understand its implications. And remember, with great power comes great responsibility. +Health Check Plugin +------------------- + +You can use the Director CLI as an Icinga CheckPlugin and monitor your Director +Health. This will run all or just one of the following test suites: + +| Name | Description | +|----------|-------------------------------------------------------------------| +| `config` | Configuration, Schema, Migrations, Deployment Endpoint | +| `sync` | All configured Sync Rules (pending changes are not a problem) | +| `import` | All configured Import Sources (pending changes are not a problem) | +| `jobs` | All configured Jobs (ignores disabled ones) | + +#### Usage + +`icingacli director clone --from [options]` + +#### Options + +| Option | Description | +|------------------|---------------------------------------| +| `--check ` | Run only a specific test suite | +| `-- ` | Use a specific Icinga Web DB resource | + +#### Examples + +```shell +icingacli director health check +``` + +```shell +icingacli director health check --check config +``` + +Sample output: + +``` +Director configuration: 5 tests OK +[OK] Database resource 'Director DB' has been specified' +[OK] Make sure the DB schema exists +[OK] There are no pending schema migrations +[OK] Deployment endpoint is 'icinga.example.com' +[OK] There is a single un-deployed change +``` + Kickstart and schema handling ----------------------------- diff --git a/doc/82-Changelog.md b/doc/82-Changelog.md index 0d5e696b..b88d741b 100644 --- a/doc/82-Changelog.md +++ b/doc/82-Changelog.md @@ -18,6 +18,9 @@ before switching to a new version. * FEATURE: Admins have now access to JSON download links in many places * FEATURE: Users equipped with related permissions can toggle "Show SQL" in the GUI +### CLI +* FEATURE: Director Health Check Plugin (#1278) + ### Import and Sync * FIX: Sync is very powerful and allows for actions not available in the GUI. It however allowed to store invalid single Service Objects with no Host. This is diff --git a/library/Director/CheckPlugin/Check.php b/library/Director/CheckPlugin/Check.php new file mode 100644 index 00000000..bba518b4 --- /dev/null +++ b/library/Director/CheckPlugin/Check.php @@ -0,0 +1,59 @@ +fail($e, $errorState); + } + + return $this; + } + + public function assertTrue($check, $message, $errorState = 'CRITICAL') + { + if ($this->makeBool($check, $message) === true) { + $this->succeed($message); + } else { + $this->fail($message, $errorState); + } + + return $this; + } + + public function assertFalse($check, $message, $errorState = 'CRITICAL') + { + if ($this->makeBool($check, $message) === false) { + $this->succeed($message); + } else { + $this->fail($message, $errorState); + } + + return $this; + } + + protected function makeBool($check, & $message) + { + if (is_callable($check)) { + try { + $check = $check(); + } catch (Exception $e) { + $message .= ': ' . $e->getMessage(); + return false; + } + } + + if (! is_bool($check)) { + return null; + } + + return $check; + } +} diff --git a/library/Director/CheckPlugin/CheckResult.php b/library/Director/CheckPlugin/CheckResult.php new file mode 100644 index 00000000..cdf9b0dc --- /dev/null +++ b/library/Director/CheckPlugin/CheckResult.php @@ -0,0 +1,31 @@ +state = $state; + } else { + $this->state = new PluginState($state); + } + + $this->output = $output; + } + + public function getState() + { + return $this->state; + } + + public function getOutput() + { + return $this->output; + } +} diff --git a/library/Director/CheckPlugin/CheckResults.php b/library/Director/CheckPlugin/CheckResults.php new file mode 100644 index 00000000..716cc578 --- /dev/null +++ b/library/Director/CheckPlugin/CheckResults.php @@ -0,0 +1,126 @@ + 0, + 1 => 0, + 2 => 0, + 3 => 0, + ]; + + public function __construct($name) + { + $this->name = $name; + $this->state = new PluginState(0); + } + + public function add(CheckResult $result) + { + $this->results[] = $result; + $this->state->raise($result->getState()); + $this->stateCounters[$result->getState()->getNumeric()]++; + + return $this; + } + + protected function getStateSummaryString() + { + $summary = [sprintf( + '%d tests OK', + $this->stateCounters[0] + )]; + + for ($i = 1; $i <= 3; $i++) { + $count = $this->stateCounters[$i]; + if ($count === 0) { + continue; + } + $summary[] = sprintf( + '%dx %s', + $count, + PluginState::create($i)->getName() + ); + } + + return implode(', ', $summary); + } + + public function getOutput() + { + $output = sprintf( + "%s: %s\n", + $this->name, + $this->getStateSummaryString() + ); + + foreach ($this->results as $result) { + $output .= sprintf( + "[%s] %s\n", + $result->getState()->getName(), + $result->getOutput() + ); + } + + return $output; + } + + public function getResults() + { + return $this->results; + } + + public function getState() + { + return $this->state; + } + + public function hasProblems() + { + return $this->getState()->getNumeric() !== 0; + } + + public function hasErrors() + { + $state = $this->getState()->getNumeric(); + return $state !== 0 && $state !== 1; + } + + public function succeed($message) + { + $this->add(new CheckResult($message)); + + return $this; + } + + public function warn($message) + { + $this->add(new CheckResult($message, 1)); + + return $this; + } + + public function fail($message, $errorState = 'CRITICAL') + { + if ($message instanceof Exception) { + $message = $message->getMessage(); + } + + $this->add(new CheckResult($message, $errorState)); + + return $this; + } +} diff --git a/library/Director/CheckPlugin/PluginState.php b/library/Director/CheckPlugin/PluginState.php new file mode 100644 index 00000000..4f9ad62f --- /dev/null +++ b/library/Director/CheckPlugin/PluginState.php @@ -0,0 +1,109 @@ + 3, + 'CRITICAL' => 2, + 'WARNING' => 1, + 'OK' => 0, + ]; + + protected static $stateNames = [ + 'OK', + 'WARNING', + 'CRITICAL', + 'UNKNOWN', + ]; + + protected static $sortSeverity = [0, 1, 3, 2]; + + /** @var int */ + protected $state; + + public function __construct($state) + { + $this->set($state); + } + + public function set($state) + { + $this->state = $this->getNumericStateFor($state); + } + + public function getNumeric() + { + return $this->state; + } + + public function getSortSeverity() + { + return static::getSortSeverityFor($this->getNumeric()); + } + + public function getName() + { + return self::$stateNames[$this->getNumeric()]; + } + + public function raise(PluginState $state) + { + if ($this->getSortSeverity() < $state->getSortSeverity()) { + $this->state = $state->getNumeric(); + } + + return $this; + } + + public static function create($state) + { + return new static($state); + } + + public static function ok() + { + return new static(0); + } + + public static function warning() + { + return new static(1); + } + + public static function critical() + { + return new static(2); + } + + public static function unknown() + { + return new static(3); + } + + protected static function getNumericStateFor($state) + { + if ((is_int($state) || ctype_digit($state)) && $state >= 0 && $state <= 3) { + return (int) $state; + } elseif (is_string($state) && array_key_exists($state, self::$stateCodes)) { + return self::$stateCodes[$state]; + } else { + throw new ProgrammingError('Expected valid state, got: %s', $state); + } + } + + protected static function getSortSeverityFor($state) + { + if (array_key_exists($state, self::$sortSeverity)) { + return self::$sortSeverity[$state]; + } else { + throw new ProgrammingError( + 'Unable to retrieve sort severity for invalid state: %s', + $state + ); + } + } +} diff --git a/library/Director/CheckPlugin/Range.php b/library/Director/CheckPlugin/Range.php new file mode 100644 index 00000000..d7b582ea --- /dev/null +++ b/library/Director/CheckPlugin/Range.php @@ -0,0 +1,101 @@ +start = $start; + $this->end = $end; + $this->mustBeWithinRange = $mustBeWithinRange; + } + + public function valueIsValid($value) + { + if ($this->valueIsWithinRange($value)) { + return $this->valueMustBeWithinRange(); + } else { + return ! $this->valueMustBeWithinRange(); + } + } + + public function valueIsWithinRange($value) + { + if ($this->start !== null && $value < $this->start) { + return false; + } + if ($this->end !== null && $value > $this->end) { + return false; + } + + return true; + } + + public function valueMustBeWithinRange() + { + return $this->mustBeWithinRange; + } + + /** + * @param $any + * @return static + */ + public static function wantRange($any) + { + if ($any instanceof static) { + return $any; + } else { + return static::parse($any); + } + } + + /** + * @param $string + * @return static + * @throws ConfigurationError + */ + public static function parse($string) + { + $string = str_replace(' ', '', $string); + $value = '[-+]?[\d\.]+'; + $valueRe = "$value(?:e$value)?"; + $regex = "/^(@)?($valueRe|~)(:$valueRe|~)?/"; + if (! preg_match($regex, $string, $match)) { + throw new ConfigurationError('Invalid range definition: %s', $string); + } + + $inside = $match[1] === '@'; + + if (strlen($match[3]) === 0) { + $start = 0; + $end = static::parseValue($match[2]); + } else { + $start = static::parseValue($match[2]); + $end = static::parseValue($match[3]); + } + $range = new static($start, $end, $inside); + + return $range; + } + + protected static function parseValue($value) + { + if ($value === '~') { + return null; + } else { + return $value; + } + } +} diff --git a/library/Director/CheckPlugin/Threshold.php b/library/Director/CheckPlugin/Threshold.php new file mode 100644 index 00000000..76aac4eb --- /dev/null +++ b/library/Director/CheckPlugin/Threshold.php @@ -0,0 +1,47 @@ +warning = Range::wantRange($warning); + } + + if ($critical !== null) { + $this->critical = Range::wantRange($critical); + } + } + + public static function check($value, $message, $warning = null, $critical = null) + { + $threshold = new static($warning, $critical); + $state = $threshold->checkValue($value); + return new CheckResult($message, $state); + } + + public function checkValue($value) + { + if ($this->critical !== null) { + if (! $this->critical->valueIsValid($value)) { + return PluginState::critical(); + } + } + + if ($this->warning !== null) { + if (! $this->warning->valueIsValid($value)) { + return PluginState::warning(); + } + } + + return PluginState::ok(); + } +} diff --git a/library/Director/Health.php b/library/Director/Health.php new file mode 100644 index 00000000..56106834 --- /dev/null +++ b/library/Director/Health.php @@ -0,0 +1,238 @@ + 'checkConfig', + 'sync' => 'checkSyncRules', + 'import' => 'checkImportSources', + 'jobs' => 'checkDirectorJobs', + ]; + + public function setDbResourceName($name) + { + $this->dbResourceName = $name; + + return $this; + } + + public function getCheck($name) + { + if (array_key_exists($name, $this->checks)) { + $func = $this->checks[$name]; + $check = $this->$func(); + } else { + $check = new CheckResults('Invalid Parameter'); + $check->fail("There is no check named '$name'"); + } + + return $check; + } + + public function getAllChecks() + { + /** @var CheckResults[] $checks */ + $checks = [$this->checkConfig()]; + + if ($checks[0]->hasErrors()) { + return $checks; + } + + $checks[] = $this->checkSyncRules(); + $checks[] = $this->checkImportSources(); + $checks[] = $this->checkDirectorJobs(); + + return $checks; + } + + protected function hasDeploymentEndpoint() + { + try { + return $this->connection->hasDeploymentEndpoint(); + } catch (Exception $e) { + return false; + } + } + + public function hasResourceConfig() + { + return $this->getDbResourceName() !== null; + } + + protected function getDbResourceName() + { + if ($this->dbResourceName === null) { + $this->dbResourceName = Config::module('director')->get('db', 'resource'); + } + + return $this->dbResourceName; + } + + protected function getConnection() + { + if ($this->connection === null) { + $this->connection = Db::fromResourceName($this->getDbResourceName()); + } + + return $this->connection; + } + + public function checkConfig() + { + $check = new Check('Director configuration'); + $name = $this->getDbResourceName(); + if ($name) { + $check->succeed("Database resource '$name' has been specified'"); + } else { + return $check->fail('No database resource has been specified'); + } + + try { + $db = $this->getConnection(); + } catch (Exception $e) { + return $check->fail($e); + } + + $migrations = new Migrations($db); + $check->assertTrue( + [$migrations, 'hasSchema'], + 'Make sure the DB schema exists' + ); + + if ($check->hasProblems()) { + return $check; + } + + $check->call(function () use ($check, $migrations) { + $count = $migrations->countPendingMigrations(); + + if ($count === 0) { + $check->succeed('There are no pending schema migrations'); + } elseif ($count === 1) { + $check->warn('There is a pending schema migration'); + } else { + $check->warn(sprintf( + 'There are %s pending schema migrations', + $count + )); + } + })->call(function () use ($check, $db) { + $check->succeed(sprintf( + "Deployment endpoint is '%s'", + $db->getDeploymentEndpointName() + )); + })->call(function () use ($check, $db) { + $count = $db->countActivitiesSinceLastDeployedConfig(); + + if ($count === 1) { + $check->succeed('There is a single un-deployed change'); + } else { + $check->succeed(sprintf( + 'There are %d un-deployed changes', + $count + )); + } + }); + + return $check; + } + + public function checkSyncRules() + { + $check = new CheckResults('Sync Rules'); + $rules = SyncRule::loadAll($this->getConnection()); + if (empty($rules)) { + $check->warn('No Sync Rules have been defined'); + return $check; + } + + foreach ($rules as $rule) { + $state = $rule->get('sync_state'); + $name = $rule->get('rule_name'); + if ($state === 'failing') { + $message = $rule->get('last_error_message'); + $check->fail("'$name' is failing: $message"); + } elseif ($state === 'pending-changes') { + $check->succeed("'$name' is fine, but there are pending changes"); + } elseif ($state === 'in-sync') { + $check->succeed("'$name' is in sync"); + } else { + $check->fail("'$name' has never been checked", 'UNKNOWN'); + } + } + + return $check; + } + + public function checkImportSources() + { + $check = new CheckResults('Import Sources'); + $sources = ImportSource::loadAll($this->getConnection()); + if (empty($sources)) { + $check->warn('No Import Sources have been defined'); + return $check; + } + + foreach ($sources as $src) { + $state = $src->get('import_state'); + $name = $src->get('source_name'); + if ($state === 'failing') { + $message = $src->get('last_error_message'); + $check->fail("'$name' is failing: $message"); + } elseif ($state === 'pending-changes') { + $check->succeed("'$name' is fine, but there are pending changes"); + } elseif ($state === 'in-sync') { + $check->succeed("'$name' is in sync"); + } else { + $check->fail("'$name' has never been checked", 'UNKNOWN'); + } + } + + return $check; + } + + public function checkDirectorJobs() + { + $check = new CheckResults('Director Jobs'); + $jobs = DirectorJob::loadAll($this->getConnection()); + if (empty($sources)) { + $check->warn('No Jobs have been defined'); + return $check; + } + + foreach ($jobs as $job) { + $name = $job->get('job_name'); + if ($job->hasBeenDisabled()) { + $check->succeed("'$name' has been disabled"); + } elseif (! $job->lastAttemptSucceeded()) { + $message = $job->get('last_error_message'); + $check->fail("Last attempt for '$name' failed: $message"); + } elseif ($job->isOverdue()) { + $check->fail("'$name' is overdue"); + } elseif ($job->shouldRun()) { + $check->succeed("'$name' is fine, but should run now"); + } else { + $check->succeed("'$name' is fine"); + } + } + + return $check; + } +} diff --git a/library/Director/Objects/DirectorJob.php b/library/Director/Objects/DirectorJob.php index fbca24f8..dc13be18 100644 --- a/library/Director/Objects/DirectorJob.php +++ b/library/Director/Objects/DirectorJob.php @@ -8,6 +8,7 @@ use Exception; class DirectorJob extends DbObjectWithSettings { + /** @var JobHook */ protected $job; protected $table = 'director_job'; @@ -68,6 +69,17 @@ class DirectorJob extends DbObjectWithSettings return (! $this->hasBeenDisabled()) && $this->isPending(); } + public function isOverdue() + { + if (! $this->shouldRun()) { + return false; + } + + return ( + strtotime($this->ts_last_attempt) + $this->run_interval * 2 + ) < time(); + } + public function hasBeenDisabled() { return $this->disabled === 'y';