CLI: health check plugin

fixes #1278
This commit is contained in:
Thomas Gelf 2017-11-08 15:31:56 +01:00
parent 6c80a20cf8
commit 9f6771f09e
11 changed files with 833 additions and 0 deletions

View File

@ -0,0 +1,62 @@
<?php
namespace Icinga\Module\Director\Clicommands;
use Icinga\Module\Director\CheckPlugin\PluginState;
use Icinga\Module\Director\Cli\Command;
use Icinga\Module\Director\Health;
/**
* Check Icinga Director Health
*
* Use this command as a CheckPlugin to monitor your Icinga Director health
*/
class HealthCommand extends Command
{
/**
* Run health checks
*
* Use this command to run all or a specific set of Health Checks.
*
* USAGE
*
* icingacli director health check [options]
*
* OPTIONS
*
* --check <name> Run only a specific set of checks
* valid names: config, sync, import, job
* --db <name> Use a specific Icinga Web DB resource
*/
public function checkAction()
{
$health = new Health();
if ($name = $this->params->get('db')) {
$health->setDbResourceName($name);
}
if ($name = $this->params->get('check')) {
$check = $health->getCheck($name);
echo $check->getOutput();
exit($check->getState()->getNumeric());
} else {
$state = new PluginState('OK');
$checks = $health->getAllChecks();
$output = [];
foreach ($checks as $check) {
$state->raise($check->getState());
$output[] = $check->getOutput();
}
if ($state === 0) {
echo "Icinga Director: everything is fine\n\n";
} else {
echo "Icinga Director: there are problems\n\n";
}
echo implode("\n", $output);
exit($state->getNumeric());
}
}
}

View File

@ -297,6 +297,51 @@ a good reason. The CLI allows you to issue operations that are not allowed in th
web frontend. Do not use this unless you really understand its implications. And
remember, with great power comes great responsibility.
Health Check Plugin
-------------------
You can use the Director CLI as an Icinga CheckPlugin and monitor your Director
Health. This will run all or just one of the following test suites:
| Name | Description |
|----------|-------------------------------------------------------------------|
| `config` | Configuration, Schema, Migrations, Deployment Endpoint |
| `sync` | All configured Sync Rules (pending changes are not a problem) |
| `import` | All configured Import Sources (pending changes are not a problem) |
| `jobs` | All configured Jobs (ignores disabled ones) |
#### Usage
`icingacli director <type> clone <name> --from <original> [options]`
#### Options
| Option | Description |
|------------------|---------------------------------------|
| `--check <name>` | Run only a specific test suite |
| `--<db> <name>` | Use a specific Icinga Web DB resource |
#### Examples
```shell
icingacli director health check
```
```shell
icingacli director health check --check config
```
Sample output:
```
Director configuration: 5 tests OK
[OK] Database resource 'Director DB' has been specified'
[OK] Make sure the DB schema exists
[OK] There are no pending schema migrations
[OK] Deployment endpoint is 'icinga.example.com'
[OK] There is a single un-deployed change
```
Kickstart and schema handling
-----------------------------

View File

@ -18,6 +18,9 @@ before switching to a new version.
* FEATURE: Admins have now access to JSON download links in many places
* FEATURE: Users equipped with related permissions can toggle "Show SQL" in the GUI
### CLI
* FEATURE: Director Health Check Plugin (#1278)
### Import and Sync
* FIX: Sync is very powerful and allows for actions not available in the GUI. It
however allowed to store invalid single Service Objects with no Host. This is

View File

@ -0,0 +1,59 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
use Exception;
class Check extends CheckResults
{
public function call(callable $check, $errorState = 'CRITICAL')
{
try {
$check();
} catch (Exception $e) {
$this->fail($e, $errorState);
}
return $this;
}
public function assertTrue($check, $message, $errorState = 'CRITICAL')
{
if ($this->makeBool($check, $message) === true) {
$this->succeed($message);
} else {
$this->fail($message, $errorState);
}
return $this;
}
public function assertFalse($check, $message, $errorState = 'CRITICAL')
{
if ($this->makeBool($check, $message) === false) {
$this->succeed($message);
} else {
$this->fail($message, $errorState);
}
return $this;
}
protected function makeBool($check, & $message)
{
if (is_callable($check)) {
try {
$check = $check();
} catch (Exception $e) {
$message .= ': ' . $e->getMessage();
return false;
}
}
if (! is_bool($check)) {
return null;
}
return $check;
}
}

View File

@ -0,0 +1,31 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
class CheckResult
{
protected $state;
protected $output;
public function __construct($output, $state = 0)
{
if ($state instanceof PluginState) {
$this->state = $state;
} else {
$this->state = new PluginState($state);
}
$this->output = $output;
}
public function getState()
{
return $this->state;
}
public function getOutput()
{
return $this->output;
}
}

View File

@ -0,0 +1,126 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
use Exception;
class CheckResults
{
/** @var string */
protected $name;
/** @var PluginState */
protected $state;
/** @var CheckResult[] */
protected $results = [];
protected $stateCounters = [
0 => 0,
1 => 0,
2 => 0,
3 => 0,
];
public function __construct($name)
{
$this->name = $name;
$this->state = new PluginState(0);
}
public function add(CheckResult $result)
{
$this->results[] = $result;
$this->state->raise($result->getState());
$this->stateCounters[$result->getState()->getNumeric()]++;
return $this;
}
protected function getStateSummaryString()
{
$summary = [sprintf(
'%d tests OK',
$this->stateCounters[0]
)];
for ($i = 1; $i <= 3; $i++) {
$count = $this->stateCounters[$i];
if ($count === 0) {
continue;
}
$summary[] = sprintf(
'%dx %s',
$count,
PluginState::create($i)->getName()
);
}
return implode(', ', $summary);
}
public function getOutput()
{
$output = sprintf(
"%s: %s\n",
$this->name,
$this->getStateSummaryString()
);
foreach ($this->results as $result) {
$output .= sprintf(
"[%s] %s\n",
$result->getState()->getName(),
$result->getOutput()
);
}
return $output;
}
public function getResults()
{
return $this->results;
}
public function getState()
{
return $this->state;
}
public function hasProblems()
{
return $this->getState()->getNumeric() !== 0;
}
public function hasErrors()
{
$state = $this->getState()->getNumeric();
return $state !== 0 && $state !== 1;
}
public function succeed($message)
{
$this->add(new CheckResult($message));
return $this;
}
public function warn($message)
{
$this->add(new CheckResult($message, 1));
return $this;
}
public function fail($message, $errorState = 'CRITICAL')
{
if ($message instanceof Exception) {
$message = $message->getMessage();
}
$this->add(new CheckResult($message, $errorState));
return $this;
}
}

View File

@ -0,0 +1,109 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
use Icinga\Exception\ProgrammingError;
class PluginState
{
protected static $stateCodes = [
'UNKNOWN' => 3,
'CRITICAL' => 2,
'WARNING' => 1,
'OK' => 0,
];
protected static $stateNames = [
'OK',
'WARNING',
'CRITICAL',
'UNKNOWN',
];
protected static $sortSeverity = [0, 1, 3, 2];
/** @var int */
protected $state;
public function __construct($state)
{
$this->set($state);
}
public function set($state)
{
$this->state = $this->getNumericStateFor($state);
}
public function getNumeric()
{
return $this->state;
}
public function getSortSeverity()
{
return static::getSortSeverityFor($this->getNumeric());
}
public function getName()
{
return self::$stateNames[$this->getNumeric()];
}
public function raise(PluginState $state)
{
if ($this->getSortSeverity() < $state->getSortSeverity()) {
$this->state = $state->getNumeric();
}
return $this;
}
public static function create($state)
{
return new static($state);
}
public static function ok()
{
return new static(0);
}
public static function warning()
{
return new static(1);
}
public static function critical()
{
return new static(2);
}
public static function unknown()
{
return new static(3);
}
protected static function getNumericStateFor($state)
{
if ((is_int($state) || ctype_digit($state)) && $state >= 0 && $state <= 3) {
return (int) $state;
} elseif (is_string($state) && array_key_exists($state, self::$stateCodes)) {
return self::$stateCodes[$state];
} else {
throw new ProgrammingError('Expected valid state, got: %s', $state);
}
}
protected static function getSortSeverityFor($state)
{
if (array_key_exists($state, self::$sortSeverity)) {
return self::$sortSeverity[$state];
} else {
throw new ProgrammingError(
'Unable to retrieve sort severity for invalid state: %s',
$state
);
}
}
}

View File

@ -0,0 +1,101 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
use Icinga\Exception\ConfigurationError;
class Range
{
/** @var float|null */
protected $start = 0;
/** @var float|null */
protected $end = null;
/** @var bool */
protected $mustBeWithinRange = true;
public function __construct($start = 0, $end = null, $mustBeWithinRange = true)
{
$this->start = $start;
$this->end = $end;
$this->mustBeWithinRange = $mustBeWithinRange;
}
public function valueIsValid($value)
{
if ($this->valueIsWithinRange($value)) {
return $this->valueMustBeWithinRange();
} else {
return ! $this->valueMustBeWithinRange();
}
}
public function valueIsWithinRange($value)
{
if ($this->start !== null && $value < $this->start) {
return false;
}
if ($this->end !== null && $value > $this->end) {
return false;
}
return true;
}
public function valueMustBeWithinRange()
{
return $this->mustBeWithinRange;
}
/**
* @param $any
* @return static
*/
public static function wantRange($any)
{
if ($any instanceof static) {
return $any;
} else {
return static::parse($any);
}
}
/**
* @param $string
* @return static
* @throws ConfigurationError
*/
public static function parse($string)
{
$string = str_replace(' ', '', $string);
$value = '[-+]?[\d\.]+';
$valueRe = "$value(?:e$value)?";
$regex = "/^(@)?($valueRe|~)(:$valueRe|~)?/";
if (! preg_match($regex, $string, $match)) {
throw new ConfigurationError('Invalid range definition: %s', $string);
}
$inside = $match[1] === '@';
if (strlen($match[3]) === 0) {
$start = 0;
$end = static::parseValue($match[2]);
} else {
$start = static::parseValue($match[2]);
$end = static::parseValue($match[3]);
}
$range = new static($start, $end, $inside);
return $range;
}
protected static function parseValue($value)
{
if ($value === '~') {
return null;
} else {
return $value;
}
}
}

View File

@ -0,0 +1,47 @@
<?php
namespace Icinga\Module\Director\CheckPlugin;
class Threshold
{
/** @var Range */
protected $warning;
/** @var Range */
protected $critical;
public function __construct($warning = null, $critical = null)
{
if ($warning !== null) {
$this->warning = Range::wantRange($warning);
}
if ($critical !== null) {
$this->critical = Range::wantRange($critical);
}
}
public static function check($value, $message, $warning = null, $critical = null)
{
$threshold = new static($warning, $critical);
$state = $threshold->checkValue($value);
return new CheckResult($message, $state);
}
public function checkValue($value)
{
if ($this->critical !== null) {
if (! $this->critical->valueIsValid($value)) {
return PluginState::critical();
}
}
if ($this->warning !== null) {
if (! $this->warning->valueIsValid($value)) {
return PluginState::warning();
}
}
return PluginState::ok();
}
}

238
library/Director/Health.php Normal file
View File

@ -0,0 +1,238 @@
<?php
namespace Icinga\Module\Director;
use Icinga\Application\Config;
use Icinga\Module\Director\CheckPlugin\Check;
use Icinga\Module\Director\CheckPlugin\CheckResults;
use Icinga\Module\Director\Db\Migrations;
use Icinga\Module\Director\Objects\DirectorJob;
use Icinga\Module\Director\Objects\ImportSource;
use Icinga\Module\Director\Objects\SyncRule;
use Exception;
class Health
{
/** @var Db */
protected $connection;
/** @var string */
protected $dbResourceName;
protected $checks = [
'config' => 'checkConfig',
'sync' => 'checkSyncRules',
'import' => 'checkImportSources',
'jobs' => 'checkDirectorJobs',
];
public function setDbResourceName($name)
{
$this->dbResourceName = $name;
return $this;
}
public function getCheck($name)
{
if (array_key_exists($name, $this->checks)) {
$func = $this->checks[$name];
$check = $this->$func();
} else {
$check = new CheckResults('Invalid Parameter');
$check->fail("There is no check named '$name'");
}
return $check;
}
public function getAllChecks()
{
/** @var CheckResults[] $checks */
$checks = [$this->checkConfig()];
if ($checks[0]->hasErrors()) {
return $checks;
}
$checks[] = $this->checkSyncRules();
$checks[] = $this->checkImportSources();
$checks[] = $this->checkDirectorJobs();
return $checks;
}
protected function hasDeploymentEndpoint()
{
try {
return $this->connection->hasDeploymentEndpoint();
} catch (Exception $e) {
return false;
}
}
public function hasResourceConfig()
{
return $this->getDbResourceName() !== null;
}
protected function getDbResourceName()
{
if ($this->dbResourceName === null) {
$this->dbResourceName = Config::module('director')->get('db', 'resource');
}
return $this->dbResourceName;
}
protected function getConnection()
{
if ($this->connection === null) {
$this->connection = Db::fromResourceName($this->getDbResourceName());
}
return $this->connection;
}
public function checkConfig()
{
$check = new Check('Director configuration');
$name = $this->getDbResourceName();
if ($name) {
$check->succeed("Database resource '$name' has been specified'");
} else {
return $check->fail('No database resource has been specified');
}
try {
$db = $this->getConnection();
} catch (Exception $e) {
return $check->fail($e);
}
$migrations = new Migrations($db);
$check->assertTrue(
[$migrations, 'hasSchema'],
'Make sure the DB schema exists'
);
if ($check->hasProblems()) {
return $check;
}
$check->call(function () use ($check, $migrations) {
$count = $migrations->countPendingMigrations();
if ($count === 0) {
$check->succeed('There are no pending schema migrations');
} elseif ($count === 1) {
$check->warn('There is a pending schema migration');
} else {
$check->warn(sprintf(
'There are %s pending schema migrations',
$count
));
}
})->call(function () use ($check, $db) {
$check->succeed(sprintf(
"Deployment endpoint is '%s'",
$db->getDeploymentEndpointName()
));
})->call(function () use ($check, $db) {
$count = $db->countActivitiesSinceLastDeployedConfig();
if ($count === 1) {
$check->succeed('There is a single un-deployed change');
} else {
$check->succeed(sprintf(
'There are %d un-deployed changes',
$count
));
}
});
return $check;
}
public function checkSyncRules()
{
$check = new CheckResults('Sync Rules');
$rules = SyncRule::loadAll($this->getConnection());
if (empty($rules)) {
$check->warn('No Sync Rules have been defined');
return $check;
}
foreach ($rules as $rule) {
$state = $rule->get('sync_state');
$name = $rule->get('rule_name');
if ($state === 'failing') {
$message = $rule->get('last_error_message');
$check->fail("'$name' is failing: $message");
} elseif ($state === 'pending-changes') {
$check->succeed("'$name' is fine, but there are pending changes");
} elseif ($state === 'in-sync') {
$check->succeed("'$name' is in sync");
} else {
$check->fail("'$name' has never been checked", 'UNKNOWN');
}
}
return $check;
}
public function checkImportSources()
{
$check = new CheckResults('Import Sources');
$sources = ImportSource::loadAll($this->getConnection());
if (empty($sources)) {
$check->warn('No Import Sources have been defined');
return $check;
}
foreach ($sources as $src) {
$state = $src->get('import_state');
$name = $src->get('source_name');
if ($state === 'failing') {
$message = $src->get('last_error_message');
$check->fail("'$name' is failing: $message");
} elseif ($state === 'pending-changes') {
$check->succeed("'$name' is fine, but there are pending changes");
} elseif ($state === 'in-sync') {
$check->succeed("'$name' is in sync");
} else {
$check->fail("'$name' has never been checked", 'UNKNOWN');
}
}
return $check;
}
public function checkDirectorJobs()
{
$check = new CheckResults('Director Jobs');
$jobs = DirectorJob::loadAll($this->getConnection());
if (empty($sources)) {
$check->warn('No Jobs have been defined');
return $check;
}
foreach ($jobs as $job) {
$name = $job->get('job_name');
if ($job->hasBeenDisabled()) {
$check->succeed("'$name' has been disabled");
} elseif (! $job->lastAttemptSucceeded()) {
$message = $job->get('last_error_message');
$check->fail("Last attempt for '$name' failed: $message");
} elseif ($job->isOverdue()) {
$check->fail("'$name' is overdue");
} elseif ($job->shouldRun()) {
$check->succeed("'$name' is fine, but should run now");
} else {
$check->succeed("'$name' is fine");
}
}
return $check;
}
}

View File

@ -8,6 +8,7 @@ use Exception;
class DirectorJob extends DbObjectWithSettings
{
/** @var JobHook */
protected $job;
protected $table = 'director_job';
@ -68,6 +69,17 @@ class DirectorJob extends DbObjectWithSettings
return (! $this->hasBeenDisabled()) && $this->isPending();
}
public function isOverdue()
{
if (! $this->shouldRun()) {
return false;
}
return (
strtotime($this->ts_last_attempt) + $this->run_interval * 2
) < time();
}
public function hasBeenDisabled()
{
return $this->disabled === 'y';