tyler.durden
/
icingaweb2-module-director
mirror of https://github.com/Icinga/icingaweb2-module-director.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/mandatory-daemon' 

fixes #1905
This commit is contained in:
Thomas Gelf 2019-09-25 12:16:14 +02:00
parent 7f79842ddf 4ee3ef2fd9
commit 7dbdb23605
39 changed files with 2230 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
application/clicommands/DaemonCommand.php Normal file
View File

@ -0,0 +1,26 @@
<?php
namespace Icinga\Module\Director\Clicommands;
use Icinga\Module\Director\Cli\Command;
use Icinga\Module\Director\Daemon\BackgroundDaemon;
class DaemonCommand extends Command
{
/**
* Run the main Director daemon
*
* USAGE
*
* icingacli director daemon run [--db-resource <name>]
*/
public function runAction()
{
$this->app->getModuleManager()->loadEnabledModules();
$daemon = new BackgroundDaemon();
if ($dbResource = $this->params->get('db-resource')) {
$daemon->setDbResourceName($dbResource);
}
$daemon->run();
}
}

114
application/clicommands/JobsCommand.php
View File

@ -2,78 +2,72 @@
namespace Icinga\Module\Director\Clicommands;
use Icinga\Module\Director\Cli\Command;
use Icinga\Module\Director\Job\JobRunner;
use Icinga\Module\Director\Objects\DirectorJob;
use Icinga\Application\Logger;
use Exception;
use gipfl\Cli\Process;
use gipfl\Protocol\JsonRpc\Connection;
use gipfl\Protocol\NetString\StreamWrapper;
use Icinga\Module\Director\Cli\Command;
use Icinga\Module\Director\Daemon\JsonRpcLogWriter as JsonRpcLogWriterAlias;
use Icinga\Module\Director\Daemon\Logger;
use Icinga\Module\Director\Objects\DirectorJob;
use React\EventLoop\Factory as Loop;
use React\EventLoop\LoopInterface;
use React\Stream\ReadableResourceStream;
use React\Stream\WritableResourceStream;
class JobsCommand extends Command
{
public function runAction()
{
$forever = $this->params->shift('forever');
if (! $forever && $this->params->getStandalone() === 'forever') {
$forever = true;
$this->params->shift();
$loop = Loop::create();
if ($this->params->get('rpc')) {
$this->enableRpc($loop);
}
$jobId = $this->params->shift();
if ($jobId) {
$this->raiseLimits();
$job = DirectorJob::loadWithAutoIncId($jobId, $this->db());
$job->run();
exit(0);
}
if ($forever) {
$this->runforever();
if ($this->params->get('rpc') && $jobId = $this->params->get('id')) {
$exitCode = 1;
$jobId = (int) $jobId;
$loop->futureTick(function () use ($jobId, $loop, &$exitCode) {
Process::setTitle('icinga::director::job');
try {
$this->raiseLimits();
$job = DirectorJob::loadWithAutoIncId($jobId, $this->db());
Process::setTitle('icinga::director::job (' . $job->get('job_name') . ')');
if ($job->run()) {
$exitCode = 0;
} else {
$exitCode = 1;
}
} catch (Exception $e) {
Logger::error($e->getMessage());
$exitCode = 1;
}
$loop->futureTick(function () use ($loop) {
$loop->stop();
});
});
} else {
$this->runAllPendingJobs();
Logger::error('This command is no longer available. Please check our Upgrading documentation');
$exitCode = 1;
}
$loop->run();
exit($exitCode);
}
protected function runforever()
protected function enableRpc(LoopInterface $loop)
{
// We'll terminate ourselves after 24h for now:
$runUnless = time() + 86400;
// stream_set_blocking(STDIN, 0);
// stream_set_blocking(STDOUT, 0);
// print_r(stream_get_meta_data(STDIN));
// stream_set_write_buffer(STDOUT, 0);
// ini_set('implicit_flush', 1);
$netString = new StreamWrapper(
new ReadableResourceStream(STDIN, $loop),
new WritableResourceStream(STDOUT, $loop)
);
$jsonRpc = new Connection();
$jsonRpc->handle($netString);
// We'll exit in case more than 100MB of memory are still in use
// after the last job execution:
$maxMem = 100 * 1024 * 1024;
while (true) {
$this->runAllPendingJobs();
if (memory_get_usage() > $maxMem) {
exit(0);
}
if (time() > $runUnless) {
exit(0);
}
sleep(2);
}
}
protected function runAllPendingJobs()
{
$jobs = new JobRunner($this->db());
try {
if ($this->hasBeenDisabled()) {
return;
}
$jobs->runPendingJobs();
} catch (Exception $e) {
Logger::error('Director Job Error: ' . $e->getMessage());
sleep(10);
}
}
protected function hasBeenDisabled()
{
return $this->db()->settings()->disable_all_jobs === 'y';
Logger::replaceRunningInstance(new JsonRpcLogWriterAlias($jsonRpc));
}
}

59
application/controllers/DaemonController.php Normal file
View File

@ -0,0 +1,59 @@
<?php
namespace Icinga\Module\Director\Controllers;
use Icinga\Application\Icinga;
use Icinga\Module\Director\Daemon\RunningDaemonInfo;
use Icinga\Module\Director\Web\Tabs\MainTabs;
use Icinga\Module\Director\Web\Controller\ActionController;
use Icinga\Module\Director\Web\Widget\BackgroundDaemonDetails;
use Icinga\Module\Director\Web\Widget\Documentation;
use ipl\Html\Html;
class DaemonController extends ActionController
{
public function indexAction()
{
$this->setAutorefreshInterval(10);
$this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('daemon');
$this->setTitle($this->translate('Director Background Daemon'));
// Avoiding layout issues:
$this->content()->add(Html::tag('h1', $this->translate('Director Background Daemon')));
// TODO: move dashboard titles into controls. Or figure out whether 2.7 "broke" this
$error = null;
try {
$db = $this->db()->getDbAdapter();
$daemons = $db->fetchAll(
$db->select()->from('director_daemon_info')->order('fqdn')->order('username')->order('pid')
);
} catch (\Exception $e) {
$daemons = [];
$error = $e->getMessage();
}
if (empty($daemons)) {
$documentation = new Documentation(Icinga::app(), $this->Auth());
$message = Html::sprintf($this->translate(
'The Icinga Director Background Daemon is not running.'
. ' Please check our %s in case you need step by step instructions'
. ' showing you how to fix this.'
), $documentation->getModuleLink(
$this->translate('documentation'),
'director',
'75-Background-Daemon',
$this->translate('Icinga Director Background Daemon')
));
$this->content()->add(Html::tag('p', ['class' => 'state-hint error'], [
$message,
($error ? [Html::tag('br'), Html::tag('strong', $error)] : ''),
]));
return;
}
foreach ($daemons as $daemon) {
$info = new RunningDaemonInfo($daemon);
$this->content()->add([new BackgroundDaemonDetails($info, $daemon) /*, $logWindow*/]);
}
}
}

30
application/controllers/DashboardController.php
View File

@ -2,6 +2,7 @@
namespace Icinga\Module\Director\Controllers;
use Icinga\Module\Director\Web\Tabs\MainTabs;
use Icinga\Module\Director\Web\Widget\HealthCheckPluginOutput;
use Icinga\Module\Director\Dashboard\Dashboard;
use Icinga\Module\Director\Health;
@ -45,22 +46,7 @@ class DashboardController extends ActionController
$dashboard = Dashboard::loadByName($name, $this->db());
$this->tabs($dashboard->getTabs())->activate($name);
} else {
$this->tabs()->add('main', [
'label' => $this->translate('Overview'),
'url' => 'director'
])->activate('main');
if ($this->hasPermission('director/admin')) {
$this->tabs()->add('health', [
'label' => $this->translate('Health'),
'url' => 'director/health'
]);
$state = $this->getHealthState();
if ($state->isProblem()) {
$this->tabs()->get('health')->setTagParams([
'class' => 'state-' . strtolower($state->getName())
]);
}
}
$this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('main');
}
$cntDashboards = 0;
@ -83,16 +69,4 @@ class DashboardController extends ActionController
$this->content()->add($msg);
}
}
/**
* @return \Icinga\Module\Director\CheckPlugin\PluginState
*/
protected function getHealthState()
{
$health = new Health();
$health->setDbResourceName($this->getDbResourceName());
$output = new HealthCheckPluginOutput($health);
return $output->getState();
}
}

10
application/controllers/HealthController.php
View File

@ -2,6 +2,7 @@
namespace Icinga\Module\Director\Controllers;
use Icinga\Module\Director\Web\Tabs\MainTabs;
use ipl\Html\Html;
use Icinga\Module\Director\Web\Widget\HealthCheckPluginOutput;
use Icinga\Module\Director\Health;
@ -12,14 +13,7 @@ class HealthController extends ActionController
public function indexAction()
{
$this->setAutorefreshInterval(10);
$this->tabs()->add('main', [
'label' => $this->translate('Overview'),
'url' => 'director'
])->add('health', [
'label' => $this->translate('Health'),
'url' => 'director/health'
])->activate('health');
$this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('health');
$this->setTitle($this->translate('Director Health'));
$health = new Health();
$health->setDbResourceName($this->getDbResourceName());

9
application/controllers/ImportsourceController.php
View File

@ -2,7 +2,6 @@
namespace Icinga\Module\Director\Controllers;
use Icinga\Exception\NotFoundError;
use Icinga\Module\Director\Forms\ImportRowModifierForm;
use Icinga\Module\Director\Forms\ImportSourceForm;
use Icinga\Module\Director\Web\ActionBar\AutomationObjectActionBar;
@ -44,6 +43,9 @@ class ImportsourceController extends ActionController
}
}
/**
* @throws \Icinga\Exception\NotFoundError
*/
protected function addMainActions()
{
$this->actions(new AutomationObjectActionBar(
@ -66,7 +68,6 @@ class ImportsourceController extends ActionController
}
/**
* @throws \Icinga\Exception\IcingaException
* @throws \Icinga\Exception\NotFoundError
*/
public function indexAction()
@ -95,7 +96,7 @@ class ImportsourceController extends ActionController
}
/**
* @throws NotFoundError
* @throws \Icinga\Exception\NotFoundError
*/
public function editAction()
{
@ -239,7 +240,7 @@ class ImportsourceController extends ActionController
/**
* @return ImportSource
* @throws NotFoundError
* @throws \Icinga\Exception\NotFoundError
*/
protected function getImportSource()
{

1
application/controllers/JobController.php
View File

@ -16,6 +16,7 @@ class JobController extends ActionController
*/
public function indexAction()
{
$this->setAutorefreshInterval(10);
$job = $this->requireJob();
$this
->addJobTabs($job, 'show')

8
application/controllers/SyncruleController.php
View File

@ -130,7 +130,7 @@ class SyncruleController extends ActionController
*/
protected function warning($msg)
{
$this->content()->add(Html::tag('p', ['class' => 'warning'], $msg));
$this->content()->add(Html::tag('p', ['class' => 'state-hint warning'], $msg));
}
/**
@ -138,7 +138,7 @@ class SyncruleController extends ActionController
*/
protected function error($msg)
{
$this->content()->add(Html::tag('p', ['class' => 'error'], $msg));
$this->content()->add(Html::tag('p', ['class' => 'state-hint error'], $msg));
}
/**
@ -165,7 +165,7 @@ class SyncruleController extends ActionController
} catch (\Exception $e) {
$this->content()->add(
Html::tag('p', [
'class' => 'error'
'class' => 'state-hint error'
], $e->getMessage())
);
@ -174,7 +174,7 @@ class SyncruleController extends ActionController
if (empty($modifications)) {
$this->content()->add(Html::tag('p', [
'class' => 'information'
'class' => 'state-hint ok'
], $this->translate('This Sync Rule is in sync and would currently not apply any changes')));
return;

21
contrib/systemd/icinga-director.service Normal file
View File

@ -0,0 +1,21 @@
[Unit]
Description=Icinga Director - Monitoring Configuration
Documentation=https://icinga.com/docs/director/latest/
Wants=network.target
[Service]
EnvironmentFile=-/etc/default/icinga-director
EnvironmentFile=-/etc/sysconfig/icinga-director
ExecStart=/usr/bin/icingacli director daemon run
ExecReload=/bin/kill -HUP ${MAINPID}
User=icingadirector
SyslogIdentifier=icingadirector
Type=notify
NotifyAccess=main
WatchdogSec=10
RestartSec=30
Restart=always
[Install]
WantedBy=multi-user.target

5
doc/05-Upgrading.md
View File

@ -54,7 +54,10 @@ Apart from this, in case you are running 1.6.x or any GIT master since then,
all you need is to replace the Director module folder with the new one. Or to
run `git checkout v1.7.x` in case you installed Director from GIT.
As always, you'll then be prompted to apply pending Database Migrations.
As always, you'll then be prompted to apply pending Database Migrations. There
is now a new, modern (and mandatory) Background Daemon, the old (optional) Jobs
Daemon must be removed. Please check our [documentation](75-Background-Daemon.md)
for related instructions.
<a name="upgrade-to-1.6.x"></a>Upgrading to 1.6.x
-------------------------------------------------

17
doc/60-CLI.md
View File

@ -593,23 +593,6 @@ with existing ones and persists eventual changes.
| `--id <id>` | A Sync Rule ID. Use the list command to figure out |
| `--benchmark` | Show timing and memory usage details |
### Running Jobs
The `jobs` command runs pending Import and Sync jobs. Please note that we have
planned a scheduler configurable through the Icinga Director web interface, but
this is not available yes.
So the only option you have right now is to trigger all jobs at once:
```shell
icingacli director jobs run
```
The output could look as follows:
```
Import "Puppet DB (PE 2015)" provides changes, triggering run... SUCCEEDED
Sync rule "Hosts from PE2015" provides changes, triggering sync... SUCCEEDED
```
Database housekeeping
---------------------

68
doc/75-Background-Daemon.md Normal file
View File

@ -0,0 +1,68 @@
<a id="Background-Daemon"></a>Background-Daemon
===============================================
The Icinga Director Background Daemon is available (and mandatory) since v1.7.0.
It is responsible for various background tasks, including fully automated Import,
Sync & Config Deployment Tasks.
Daemon Installation
-------------------
In case you installed Icinga Director as a package, the daemon should already
have been installed. In case you're running directly from a GIT working copy or
from a manual installation, you need to tell `systemd` about your new service.
First make sure that the system user `icingadirector` exists. In case it doesn't,
please create one:
```sh
useradd -r -g icingaweb2 -d /var/lib/icingadirector -s /bin/false icingadirector
install -d -o icingadirector -g icingaweb2 -m 0750 /var/lib/icingadirector
```
Then copy the provided Unit-File from our [contrib](../contrib/systemd/icinga-director.service)
to `/etc/systemd/system`, enable and start the service:
```sh
MODULE_PATH=/usr/share/icingaweb2/modules/director
cp "${MODULE_PATH}/contrib/systemd/icinga-director.service" /etc/systemd/system/
systemctl daemon-reload
```
Now your system knows about the Icinga Director Daemon. You should make sure that
it starts automatically each time your system boots:
```sh
systemctl enable icinga-director.service
```
Starting the Daemon
-------------------
You now can start the Background daemon like any other service on your Linux system:
```sh
systemctl enable icinga-director.service
```
Stopping the Daemon
-------------------
You now can start the Background daemon like any other service on your Linux system:
```sh
systemctl enable icinga-director.service
```
Getting rid of the old Job Daemon
---------------------------------
Before v1.7.0, Icinga Director shipped an optional Job Daemon. This one is no longer
needed and should be removed from your system as follows:
```sh
systemctl stop director-jobs
systemctl disable director-jobs
rm /etc/systemd/system/director-jobs.service
systemctl daemon-reload
```

212
library/Director/Daemon/BackgroundDaemon.php Normal file
View File

@ -0,0 +1,212 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Exception;
use gipfl\Cli\Process;
use gipfl\IcingaCliDaemon\DbResourceConfigWatch;
use gipfl\SystemD\NotifySystemD;
use Icinga\Module\Director\Db;
use React\EventLoop\Factory as Loop;
use React\EventLoop\LoopInterface;
use Ramsey\Uuid\Uuid;
class BackgroundDaemon
{
/** @var LoopInterface */
private $loop;
/** @var Db */
protected $connection;
/** @var NotifySystemD|boolean */
protected $systemd;
protected $onShutdown;
/** @var JobRunner */
protected $jobRunner;
/** @var string|null */
protected $dbResourceName;
/** @var DaemonDb */
protected $daemonDb;
/** @var DaemonProcessState */
protected $processState;
/** @var DaemonProcessDetails */
protected $processDetails;
/** @var LogProxy */
protected $logProxy;
/** @var bool */
protected $reloading = false;
/** @var bool */
protected $shuttingDown = false;
public function run(LoopInterface $loop = null)
{
if ($ownLoop = $loop === null) {
$loop = Loop::create();
}
$this->loop = $loop;
$this->loop->futureTick(function () {
$this->initialize();
});
if ($ownLoop) {
$loop->run();
}
}
public function setDbResourceName($name)
{
$this->dbResourceName = $name;
return $this;
}
protected function initialize()
{
$this->registerSignalHandlers($this->loop);
$this->processState = new DaemonProcessState('icinga::director');
$this->jobRunner = new JobRunner($this->loop);
$this->systemd = $this->eventuallyInitializeSystemd();
$this->processState->setSystemd($this->systemd);
$this->processDetails = $this
->initializeProcessDetails($this->systemd)
->registerProcessList($this->jobRunner->getProcessList());
$this->logProxy = new LogProxy($this->processDetails->getInstanceUuid());
$this->jobRunner->forwardLog($this->logProxy);
$this->daemonDb = $this->initializeDb(
$this->processDetails,
$this->processState,
$this->dbResourceName
);
$this->daemonDb
->register($this->jobRunner)
->register($this->logProxy)
->run($this->loop);
if ($this->systemd) {
$this->systemd->setReady();
}
}
/**
* @param NotifySystemD|false $systemd
* @return DaemonProcessDetails
*/
protected function initializeProcessDetails($systemd)
{
if ($systemd && $systemd->hasInvocationId()) {
$uuid = $systemd->getInvocationId();
} else {
try {
$uuid = \bin2hex(Uuid::uuid4()->getBytes());
} catch (Exception $e) {
$uuid = 'deadc0de' . \substr(\md5(\getmypid()), 0, 24);
}
}
$processDetails = new DaemonProcessDetails($uuid);
if ($systemd) {
$processDetails->set('running_with_systemd', 'y');
}
return $processDetails;
}
protected function eventuallyInitializeSystemd()
{
$systemd = NotifySystemD::ifRequired($this->loop);
if ($systemd) {
Logger::replaceRunningInstance(new SystemdLogWriter());
Logger::info(sprintf(
"Started by systemd, notifying watchdog every %0.2Gs via %s",
$systemd->getWatchdogInterval(),
$systemd->getSocketPath()
));
} else {
Logger::debug('Running without systemd');
}
return $systemd;
}
protected function initializeDb(
DaemonProcessDetails $processDetails,
DaemonProcessState $processState,
$dbResourceName = null
) {
$db = new DaemonDb($processDetails);
$db->on('state', function ($state) use ($processState) {
$processState->setComponentState('db', $state);
});
$db->setConfigWatch(
$dbResourceName
? DbResourceConfigWatch::name($dbResourceName)
: DbResourceConfigWatch::module('director')
);
return $db;
}
protected function registerSignalHandlers(LoopInterface $loop)
{
$func = function ($signal) use (&$func) {
$this->shutdownWithSignal($signal, $func);
};
$funcReload = function () {
$this->reload();
};
$loop->addSignal(SIGHUP, $funcReload);
$loop->addSignal(SIGINT, $func);
$loop->addSignal(SIGTERM, $func);
}
protected function shutdownWithSignal($signal, &$func)
{
$this->loop->removeSignal($signal, $func);
$this->shutdown();
}
protected function reload()
{
if ($this->reloading) {
Logger::error('Ignoring reload request, reload is already in progress');
return;
}
$this->reloading = true;
$this->setState('reloading the main process');
$this->daemonDb->disconnect()->then(function () {
Process::restart();
});
}
protected function shutdown()
{
if ($this->shuttingDown) {
Logger::error('Ignoring shutdown request, shutdown is already in progress');
return;
}
Logger::info('Shutting down');
$this->shuttingDown = true;
$this->setState('shutting down');
$this->daemonDb->disconnect()->then(function () {
Logger::info('DB has been disconnected, shutdown finished');
$this->loop->stop();
});
}
protected function setState($state)
{
if ($this->processState) {
$this->processState->setState($state);
}
return $this;
}
}

61
library/Director/Daemon/BackgroundDaemonState.php Normal file
View File

@ -0,0 +1,61 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Icinga\Module\Director\Db;
class BackgroundDaemonState
{
protected $db;
/** @var RunningDaemonInfo[] */
protected $instances;
public function __construct(Db $db)
{
$this->db = $db;
}
public function hasProblems()
{
return $this->isRunning();
}
public function isRunning()
{
foreach ($this->getInstances() as $instance) {
if ($instance->isRunning()) {
return true;
}
}
return false;
}
public function getInstances()
{
if ($this->instances === null) {
$this->instances = $this->fetchInfo();
}
return $this->instances;
}
/**
* @return RunningDaemonInfo[]
*/
protected function fetchInfo()
{
$db = $this->db->getDbAdapter();
$daemons = $db->fetchAll(
$db->select()->from('director_daemon_info')->order('fqdn')->order('username')->order('pid')
);
$result = [];
foreach ($daemons as $info) {
$result[] = new RunningDaemonInfo($info);
}
return $result;
}
}

317
library/Director/Daemon/DaemonDb.php Normal file
View File

@ -0,0 +1,317 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Exception;
use gipfl\IcingaCliDaemon\DbResourceConfigWatch;
use gipfl\IcingaCliDaemon\RetryUnless;
use Icinga\Data\ConfigObject;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Db\Migrations;
use ipl\Stdlib\EventEmitter;
use React\EventLoop\LoopInterface;
use React\Promise\Deferred;
use React\Promise\FulfilledPromise;
use React\Promise\RejectedPromise;
use RuntimeException;
use SplObjectStorage;
class DaemonDb
{
use EventEmitter;
/** @var LoopInterface */
private $loop;
/** @var Db */
protected $connection;
/** @var \Zend_Db_Adapter_Abstract */
protected $db;
/** @var DaemonProcessDetails */
protected $details;
/** @var DbBasedComponent[] */
protected $registeredComponents = [];
/** @var DbResourceConfigWatch|null */
protected $configWatch;
/** @var array|null */
protected $dbConfig;
/** @var RetryUnless|null */
protected $pendingReconnection;
/** @var Deferred|null */
protected $pendingDisconnect;
protected $refreshTimer;
public function __construct(DaemonProcessDetails $details, $dbConfig = null)
{
$this->details = $details;
$this->dbConfig = $dbConfig;
}
public function register(DbBasedComponent $component)
{
$this->registeredComponents[] = $component;
return $this;
}
public function setConfigWatch(DbResourceConfigWatch $configWatch)
{
$this->configWatch = $configWatch;
$configWatch->notify(function ($config) {
$this->disconnect()->then(function () use ($config) {
return $this->onNewConfig($config);
});
});
if ($this->loop) {
$configWatch->run($this->loop);
}
return $this;
}
public function run(LoopInterface $loop)
{
$this->loop = $loop;
$this->connect();
$this->refreshTimer = $loop->addPeriodicTimer(3, function () {
$this->refreshMyState();
});
if ($this->configWatch) {
$this->configWatch->run($this->loop);
}
}
protected function onNewConfig($config)
{
if ($config === null) {
if ($this->dbConfig === null) {
Logger::error('DB configuration is not valid');
} else {
Logger::error('DB configuration is no longer valid');
}
$this->emitStatus('there is no valid DB configuration');
$this->dbConfig = $config;
return new FulfilledPromise();
} else {
$this->emitStatus('configuration loaded');
$this->dbConfig = $config;
return $this->establishConnection($config);
}
}
protected function establishConnection($config)
{
if ($this->connection !== null) {
Logger::error('Trying to establish a connection while being connected');
return new RejectedPromise();
}
$callback = function () use ($config) {
$this->reallyEstablishConnection($config);
};
$onSuccess = function () {
$this->pendingReconnection = null;
$this->onConnected();
};
if ($this->pendingReconnection) {
$this->pendingReconnection->reset();
$this->pendingReconnection = null;
}
return $this->pendingReconnection = RetryUnless::succeeding($callback)
->setInterval(0.2)
->slowDownAfter(10, 10)
->run($this->loop)
->then($onSuccess)
;
}
protected function reallyEstablishConnection($config)
{
$connection = new Db(new ConfigObject($config));
$connection->getDbAdapter()->getConnection();
$migrations = new Migrations($connection);
if (! $migrations->hasSchema()) {
$this->emit('status', ['DB has no schema', 'error']);
throw new RuntimeException('DB has no schema');
}
$this->wipeOrphanedInstances($connection);
if ($this->hasAnyOtherActiveInstance($connection)) {
throw new RuntimeException('DB is locked by a running daemon instance');
}
$this->details->set('schema_version', $migrations->getLastMigrationNumber());
$this->connection = $connection;
$this->db = $connection->getDbAdapter();
$this->loop->futureTick(function () {
$this->refreshMyState();
});
return $connection;
}
protected function onConnected()
{
$this->emitStatus('connected');
Logger::info('Connected to the database');
foreach ($this->registeredComponents as $component) {
$component->initDb($this->connection);
}
}
/**
* @return \React\Promise\PromiseInterface
*/
protected function reconnect()
{
return $this->disconnect()->then(function () {
return $this->connect();
}, function (Exception $e) {
Logger::error('Disconnect failed. This should never happen: ' . $e->getMessage());
exit(1);
});
}
/**
* @return \React\Promise\ExtendedPromiseInterface
*/
public function connect()
{
if ($this->connection === null) {
if ($this->dbConfig) {
return $this->establishConnection($this->dbConfig);
}
}
return new FulfilledPromise();
}
/**
* @return \React\Promise\ExtendedPromiseInterface
*/
public function disconnect()
{
if (! $this->connection) {
return new FulfilledPromise();
}
if ($this->pendingDisconnect) {
return $this->pendingDisconnect->promise();
}
$this->eventuallySetStopped();
$this->pendingDisconnect = new Deferred();
$pendingComponents = new SplObjectStorage();
foreach ($this->registeredComponents as $component) {
$pendingComponents->attach($component);
$resolve = function () use ($pendingComponents, $component) {
$pendingComponents->detach($component);
if ($pendingComponents->count() === 0) {
$this->pendingDisconnect->resolve();
}
};
// TODO: What should we do in case they don't?
$component->stopDb()->then($resolve);
}
try {
if ($this->db) {
$this->db->closeConnection();
}
} catch (Exception $e) {
Logger::error('Failed to disconnect: ' . $e->getMessage());
}
return $this->pendingDisconnect->promise()->then(function () {
$this->connection = null;
$this->db = null;
$this->pendingDisconnect = null;
});
}
protected function emitStatus($message, $level = 'info')
{
$this->emit('status', [$message, $level]);
return $this;
}
protected function hasAnyOtherActiveInstance(Db $connection)
{
$db = $connection->getDbAdapter();
return (int) $db->fetchOne(
$db->select()
->from('director_daemon_info', 'COUNT(*)')
->where('ts_stopped IS NULL')
) > 0;
}
protected function wipeOrphanedInstances(Db $connection)
{
$db = $connection->getDbAdapter();
$db->delete('director_daemon_info', 'ts_stopped IS NOT NULL');
$db->delete('director_daemon_info', $db->quoteInto(
'instance_uuid_hex = ?',
$this->details->getInstanceUuid()
));
$count = $db->delete(
'director_daemon_info',
'ts_stopped IS NULL AND ts_last_update < ' . (
DaemonUtil::timestampWithMilliseconds() - (60 * 1000)
)
);
if ($count > 1) {
Logger::error("Removed $count orphaned daemon instance(s) from DB");
}
}
protected function refreshMyState()
{
if ($this->db === null || $this->pendingReconnection || $this->pendingDisconnect) {
return;
}
try {
$updated = $this->db->update(
'director_daemon_info',
$this->details->getPropertiesToUpdate(),
$this->db->quoteInto('instance_uuid_hex = ?', $this->details->getInstanceUuid())
);
if (! $updated) {
$this->db->insert(
'director_daemon_info',
$this->details->getPropertiesToInsert()
);
}
} catch (Exception $e) {
Logger::error($e->getMessage());
$this->reconnect();
}
}
protected function eventuallySetStopped()
{
try {
if (! $this->db) {
return;
}
$this->db->update(
'director_daemon_info',
['ts_stopped' => DaemonUtil::timestampWithMilliseconds()],
$this->db->quoteInto('instance_uuid_hex = ?', $this->details->getInstanceUuid())
);
} catch (Exception $e) {
Logger::error('Failed to update daemon info (setting ts_stopped): ' . $e->getMessage());
}
}
}

122
library/Director/Daemon/DaemonProcessDetails.php Normal file
View File

@ -0,0 +1,122 @@
<?php
namespace Icinga\Module\Director\Daemon;
use gipfl\LinuxHealth\Memory;
use Icinga\Application\Platform;
use React\ChildProcess\Process;
use gipfl\Cli\Process as CliProcess;
class DaemonProcessDetails
{
/** @var string */
protected $instanceUuid;
/** @var \stdClass */
protected $info;
/** @var ProcessList[] */
protected $processLists = [];
protected $myArgs;
protected $myPid;
public function __construct($instanceUuid)
{
$this->instanceUuid = $instanceUuid;
$this->initialize();
}
public function getInstanceUuid()
{
return $this->instanceUuid;
}
public function getPropertiesToInsert()
{
return $this->getPropertiesToUpdate() + (array) $this->info;
}
public function getPropertiesToUpdate()
{
return [
'ts_last_update' => DaemonUtil::timestampWithMilliseconds(),
'ts_stopped' => null,
'process_info' => \json_encode($this->collectProcessInfo()),
];
}
public function set($property, $value)
{
if (\property_exists($this->info, $property)) {
$this->info->$property = $value;
} else {
throw new \InvalidArgumentException("Trying to set invalid daemon info property: $property");
}
}
public function registerProcessList(ProcessList $list)
{
$refresh = function (Process $process) {
$this->refreshProcessInfo();
};
$list->on('start', $refresh)->on('exit', $refresh);
$this->processLists[] = $list;
return $this;
}
protected function refreshProcessInfo()
{
$this->set('process_info', \json_encode($this->collectProcessInfo()));
}
protected function collectProcessInfo()
{
$info = (object) [$this->myPid => (object) [
'command' => implode(' ', $this->myArgs),
'running' => true,
'memory' => Memory::getUsageForPid($this->myPid)
]];
foreach ($this->processLists as $processList) {
foreach ($processList->getOverview() as $pid => $details) {
$info->$pid = $details;
}
}
return $info;
}
protected function initialize()
{
global $argv;
CliProcess::getInitialCwd();
$this->myArgs = $argv;
$this->myPid = \posix_getpid();
if (isset($_SERVER['_'])) {
$self = $_SERVER['_'];
} else {
// Process does a better job, but want the relative path (if such)
$self = $_SERVER['PHP_SELF'];
}
$this->info = (object) [
'instance_uuid_hex' => $this->instanceUuid,
'running_with_systemd' => 'n',
'ts_started' => (int) ((float) $_SERVER['REQUEST_TIME_FLOAT'] * 1000),
'ts_stopped' => null,
'pid' => \posix_getpid(),
'fqdn' => Platform::getFqdn(),
'username' => Platform::getPhpUser(),
'schema_version' => null,
'php_version' => Platform::getPhpVersion(),
'binary_path' => $self,
'binary_realpath' => CliProcess::getBinaryPath(),
'php_integer_size' => PHP_INT_SIZE,
'php_binary_path' => PHP_BINARY,
'php_binary_realpath' => \realpath(PHP_BINARY), // TODO: useless?
'process_info' => null,
];
}
}

85
library/Director/Daemon/DaemonProcessState.php Normal file
View File

@ -0,0 +1,85 @@
<?php
namespace Icinga\Module\Director\Daemon;
use gipfl\Cli\Process;
use gipfl\SystemD\NotifySystemD;
class DaemonProcessState
{
/** @var NotifySystemD|null */
protected $systemd;
protected $components = [];
protected $currentMessage;
protected $processTitle;
protected $state;
public function __construct($processTitle)
{
$this->processTitle = $processTitle;
$this->refreshMessage();
}
/**
* @param NotifySystemD|false $systemd
* @return $this
*/
public function setSystemd($systemd)
{
if ($systemd) {
$this->systemd = $systemd;
} else {
$this->systemd = null;
}
return $this;
}
public function setState($message)
{
$this->state = $message;
$this->refreshMessage();
return $this;
}
public function setComponentState($name, $stateMessage)
{
if ($stateMessage === null) {
unset($this->components[$name]);
} else {
$this->components[$name] = $stateMessage;
}
$this->refreshMessage();
}
protected function refreshMessage()
{
$messageParts = [];
if (\strlen($this->state)) {
$messageParts[] = $this->state;
}
foreach ($this->components as $component => $message) {
$messageParts[] = "$component: $message";
}
$message = \implode(', ', $messageParts);
if ($message !== $this->currentMessage) {
$this->currentMessage = $message;
if (\strlen($message) === 0) {
Process::setTitle($this->processTitle);
} else {
Process::setTitle($this->processTitle . ": $message");
}
if ($this->systemd) {
$this->systemd->setStatus($message);
}
}
}
}

16
library/Director/Daemon/DaemonUtil.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace Icinga\Module\Director\Daemon;
class DaemonUtil
{
/**
* @return int
*/
public static function timestampWithMilliseconds()
{
$mTime = explode(' ', microtime());
return (int) round($mTime[0] * 1000) + (int) $mTime[1] * 1000;
}
}

19
library/Director/Daemon/DbBasedComponent.php Normal file
View File

@ -0,0 +1,19 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Icinga\Module\Director\Db;
interface DbBasedComponent
{
/**
* @param Db $db
* @return \React\Promise\ExtendedPromiseInterface;
*/
public function initDb(Db $db);
/**
* @return \React\Promise\ExtendedPromiseInterface;
*/
public function stopDb();
}

234
library/Director/Daemon/JobRunner.php Normal file
View File

@ -0,0 +1,234 @@
<?php
namespace Icinga\Module\Director\Daemon;
use gipfl\IcingaCliDaemon\FinishedProcessState;
use gipfl\IcingaCliDaemon\IcingaCliRpc;
use Icinga\Application\Logger;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Objects\DirectorJob;
use React\ChildProcess\Process;
use React\EventLoop\LoopInterface;
use React\Promise\FulfilledPromise;
use React\Promise\Promise;
class JobRunner implements DbBasedComponent
{
/** @var Db */
protected $db;
/** @var LoopInterface */
protected $loop;
/** @var int[] */
protected $scheduledIds = [];
/** @var Promise[] */
protected $runningIds = [];
protected $checkInterval = 10;
/** @var \React\EventLoop\TimerInterface */
protected $timer;
/** @var LogProxy */
protected $logProxy;
/** @var ProcessList */
protected $running;
public function __construct(LoopInterface $loop)
{
$this->loop = $loop;
$this->running = new ProcessList($loop);
}
public function forwardLog(LogProxy $logProxy)
{
$this->logProxy = $logProxy;
return $this;
}
/**
* @param Db $db
* @return \React\Promise\ExtendedPromiseInterface
*/
public function initDb(Db $db)
{
$this->db = $db;
$check = function () {
try {
$this->checkForPendingJobs();
$this->runNextPendingJob();
} catch (\Exception $e) {
Logger::error($e->getMessage());
}
};
if ($this->timer === null) {
$this->loop->futureTick($check);
}
if ($this->timer !== null) {
Logger::info('Cancelling former timer');
$this->loop->cancelTimer($this->timer);
}
$this->timer = $this->loop->addPeriodicTimer($this->checkInterval, $check);
return new FulfilledPromise();
}
/**
* @return \React\Promise\ExtendedPromiseInterface
*/
public function stopDb()
{
$this->scheduledIds = [];
if ($this->timer !== null) {
$this->loop->cancelTimer($this->timer);
$this->timer = null;
}
$allFinished = $this->running->killOrTerminate();
foreach ($this->runningIds as $id => $promise) {
$promise->cancel();
}
$this->runningIds = [];
return $allFinished;
}
protected function hasBeenDisabled()
{
$db = $this->db->getDbAdapter();
return $db->fetchOne(
$db->select()
->from('director_setting', 'setting_value')
->where('setting_name = ?', 'disable_all_jobs')
) === 'y';
}
protected function checkForPendingJobs()
{
if ($this->hasBeenDisabled()) {
$this->scheduledIds = [];
// TODO: disable jobs currently going on?
return;
}
if (empty($this->scheduledIds)) {
$this->loadNextIds();
}
}
protected function runNextPendingJob()
{
if ($this->timer === null) {
// Reset happened. Stopping?
return;
}
if (! empty($this->runningIds)) {
return;
}
while (! empty($this->scheduledIds)) {
if ($this->runNextJob()) {
break;
}
}
}
protected function loadNextIds()
{
$db = $this->db->getDbAdapter();
foreach ($db->fetchCol(
$db->select()->from('director_job', 'id')->where('disabled = ?', 'n')
) as $id) {
$this->scheduledIds[] = (int) $id;
};
}
/**
* @return bool
*/
protected function runNextJob()
{
$id = \array_shift($this->scheduledIds);
try {
$job = DirectorJob::loadWithAutoIncId((int) $id, $this->db);
if ($job->shouldRun()) {
$this->runJob($job);
return true;
}
} catch (\Exception $e) {
Logger::error('Trying to schedule Job failed: ' . $e->getMessage());
}
return false;
}
/**
* @param DirectorJob $job
*/
protected function runJob(DirectorJob $job)
{
$id = $job->get('id');
$jobName = $job->get('job_name');
Logger::debug("Job starting: $jobName");
$arguments = [
'director',
'job',
'run',
'--id',
$job->get('id'),
'--debug',
'--rpc'
];
$cli = new IcingaCliRpc();
$cli->setArguments($arguments);
$cli->on('start', function (Process $process) {
$this->onProcessStarted($process);
});
// Happens on protocol (Netstring) errors or similar:
$cli->on('error', function (\Exception $e) {
Logger::error('UNEXPECTED: ' . rtrim($e->getMessage()));
});
if ($this->logProxy) {
$logger = clone($this->logProxy);
$logger->setPrefix("[$jobName]: ");
$cli->rpc()->setHandler($this->logProxy, 'logger');
}
unset($this->scheduledIds[$id]);
$this->runningIds[$id] = $cli->run($this->loop)->then(function () use ($id, $jobName) {
Logger::debug("Job finished: $jobName");
})->otherwise(function (\Exception $e) use ($id, $jobName) {
Logger::error('Job failed: ' . $e->getMessage());
})->otherwise(function (FinishedProcessState $state) {
Logger::error($state->getReason());
})->always(function () use ($id) {
unset($this->runningIds[$id]);
$this->loop->futureTick(function () {
$this->runNextPendingJob();
});
});
}
/**
* @return ProcessList
*/
public function getProcessList()
{
return $this->running;
}
protected function onProcessStarted(Process $process)
{
$this->running->attach($process);
}
public function __destruct()
{
$this->stopDb();
$this->logProxy = null;
$this->loop = null;
}
}

37
library/Director/Daemon/JsonRpcLogWriter.php Normal file
View File

@ -0,0 +1,37 @@
<?php
namespace Icinga\Module\Director\Daemon;
use gipfl\Protocol\JsonRpc\Connection;
use gipfl\Protocol\JsonRpc\Notification;
use Icinga\Application\Logger\LogWriter;
use Icinga\Data\ConfigObject;
class JsonRpcLogWriter extends LogWriter
{
protected $connection;
protected static $severityMap = [
Logger::DEBUG => 'debug',
Logger::INFO => 'info',
Logger::WARNING => 'warning',
Logger::ERROR => 'error',
];
public function __construct(Connection $connection)
{
parent::__construct(new ConfigObject([]));
$this->connection = $connection;
}
public function log($severity, $message)
{
$message = \iconv('UTF-8', 'UTF-8//IGNORE', $message);
$this->connection->sendNotification(
Notification::create('logger.log', [
static::$severityMap[$severity],
$message
])
);
}
}

76
library/Director/Daemon/LogProxy.php Normal file
View File

@ -0,0 +1,76 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Exception;
use Icinga\Module\Director\Db;
use React\Promise\FulfilledPromise;
class LogProxy implements DbBasedComponent
{
protected $connection;
protected $db;
protected $server;
protected $instanceUuid;
protected $prefix = '';
public function __construct($instanceUuid)
{
$this->instanceUuid = $instanceUuid;
}
public function setPrefix($prefix)
{
$this->prefix = $prefix;
return $this;
}
/**
* @param Db $connection
* @return \React\Promise\ExtendedPromiseInterface
*/
public function initDb(Db $connection)
{
$this->connection = $connection;
$this->db = $connection->getDbAdapter();
return new FulfilledPromise();
}
/**
* @return \React\Promise\ExtendedPromiseInterface
*/
public function stopDb()
{
$this->connection = null;
$this->db = null;
return new FulfilledPromise();
}
public function log($severity, $message)
{
Logger::$severity($this->prefix . $message);
/*
// Not yet
try {
if ($this->db) {
$this->db->insert('director_daemonlog', [
// environment/installation/db?
'instance_uuid' => $this->instanceUuid,
'ts_create' => DaemonUtil::timestampWithMilliseconds(),
'level' => $severity,
'message' => $message,
]);
}
} catch (Exception $e) {
Logger::error($e->getMessage());
}
*/
}
}

21
library/Director/Daemon/Logger.php Normal file
View File

@ -0,0 +1,21 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Icinga\Application\Logger as IcingaLogger;
use Icinga\Application\Logger\LogWriter;
use Icinga\Exception\ConfigurationError;
class Logger extends IcingaLogger
{
public static function replaceRunningInstance(LogWriter $writer, $level = self::DEBUG)
{
try {
self::$instance
->setLevel($level)
->writer = $writer;
} catch (ConfigurationError $e) {
self::$instance->error($e->getMessage());
}
}
}

125
library/Director/Daemon/ProcessList.php Normal file
View File

@ -0,0 +1,125 @@
<?php
namespace Icinga\Module\Director\Daemon;
use gipfl\LinuxHealth\Memory;
use Icinga\Application\Logger;
use ipl\Stdlib\EventEmitter;
use React\ChildProcess\Process;
use React\EventLoop\LoopInterface;
use React\Promise\Deferred;
use React\Promise\FulfilledPromise;
class ProcessList
{
use EventEmitter;
/** @var LoopInterface */
protected $loop;
/** @var \SplObjectStorage */
protected $processes;
/**
* ProcessList constructor.
* @param LoopInterface $loop
* @param Process[] $processes
*/
public function __construct(LoopInterface $loop, array $processes = [])
{
$this->loop = $loop;
$this->processes = new \SplObjectStorage();
foreach ($processes as $process) {
$this->attach($process);
}
}
public function attach(Process $process)
{
$this->processes->attach($process);
$this->emit('start', [$process]);
$process->on('exit', function () use ($process) {
$this->detach($process);
$this->emit('exit', [$process]);
});
return $this;
}
public function detach(Process $process)
{
$this->processes->detach($process);
return $this;
}
/**
* @param int $timeout
* @return \React\Promise\ExtendedPromiseInterface
*/
public function killOrTerminate($timeout = 5)
{
if ($this->processes->count() === 0) {
return new FulfilledPromise();
}
$deferred = new Deferred();
$killTimer = $this->loop->addTimer($timeout, function () use ($deferred) {
/** @var Process $process */
foreach ($this->processes as $process) {
$pid = $process->getPid();
Logger::error("Process $pid is still running, sending SIGKILL");
$process->terminate(SIGKILL);
}
// Let's a little bit of delay after KILLing
$this->loop->addTimer(0.1, function () use ($deferred) {
$deferred->resolve();
});
});
$timer = $this->loop->addPeriodicTimer($timeout / 20, function () use (
$deferred,
& $timer,
$killTimer
) {
$stopped = [];
/** @var Process $process */
foreach ($this->processes as $process) {
if (! $process->isRunning()) {
$stopped[] = $process;
}
}
foreach ($stopped as $process) {
$this->processes->detach($process);
}
if ($this->processes->count() === 0) {
$this->loop->cancelTimer($timer);
$this->loop->cancelTimer($killTimer);
$deferred->resolve();
}
});
/** @var Process $process */
foreach ($this->processes as $process) {
$process->terminate(SIGTERM);
}
return $deferred->promise();
}
public function getOverview()
{
$info = [];
/** @var Process $process */
foreach ($this->processes as $process) {
$pid = $process->getPid();
$info[$pid] = (object) [
'command' => preg_replace('/^exec /', '', $process->getCommand()),
'running' => $process->isRunning(),
'memory' => Memory::getUsageForPid($pid)
];
}
return $info;
}
}

154
library/Director/Daemon/RunningDaemonInfo.php Normal file
View File

@ -0,0 +1,154 @@
<?php
namespace Icinga\Module\Director\Daemon;
class RunningDaemonInfo
{
/** @var object */
protected $info;
public function __construct($info = null)
{
$this->setInfo($info);
}
public function setInfo($info)
{
if (empty($info)) {
$this->info = $this->createEmptyInfo();
} else {
$this->info = $info;
}
return $this;
}
public function isRunning()
{
return $this->getPid() !== null && ! $this->isOutdated();
}
public function getPid()
{
return (int) $this->info->pid;
}
public function getUsername()
{
return $this->info->username;
}
public function getFqdn()
{
return $this->info->fqdn;
}
public function getLastUpdate()
{
return $this->info->ts_last_update;
}
public function getLastModification()
{
return $this->info->ts_last_modification;
}
public function getPhpVersion()
{
return $this->info->php_version;
}
public function hasBeenStopped()
{
return $this->getTimestampStopped() !== null;
}
public function getTimestampStarted()
{
return $this->info->ts_started;
}
public function getTimestampStopped()
{
return $this->info->ts_stopped;
}
public function isOutdated($seconds = 5)
{
return (
DaemonUtil::timestampWithMilliseconds() - $this->info->ts_last_update
) > $seconds * 1000;
}
public function isRunningWithSystemd()
{
return $this->info->running_with_systemd === 'y';
}
public function getBinaryPath()
{
return $this->info->binary_path;
}
public function getBinaryRealpath()
{
return $this->info->binary_realpath;
}
public function binaryRealpathDiffers()
{
return $this->getBinaryPath() !== $this->getBinaryRealpath();
}
public function getPhpBinaryPath()
{
return $this->info->php_binary_path;
}
public function getPhpBinaryRealpath()
{
return $this->info->php_binary_realpath;
}
public function phpBinaryRealpathDiffers()
{
return $this->getPhpBinaryPath() !== $this->getPhpBinaryRealpath();
}
public function getPhpIntegerSize()
{
return (int) $this->info->php_integer_size;
}
public function has64bitIntegers()
{
return $this->getPhpIntegerSize() === 8;
}
/*
// TODO: not yet
public function isMaster()
{
return $this->info->is_master === 'y';
}
public function isStandby()
{
return ! $this->isMaster();
}
*/
protected function createEmptyInfo()
{
return (object) [
'pid' => null,
'fqdn' => null,
'username' => null,
'php_version' => null,
// 'is_master' => null,
// Only if not running. Does this make any sense in 'empty info'?
'ts_last_update' => null,
'ts_last_modification' => null
];
}
}

27
library/Director/Daemon/SystemdLogWriter.php Normal file
View File

@ -0,0 +1,27 @@
<?php
namespace Icinga\Module\Director\Daemon;
use Icinga\Application\Logger\LogWriter;
use Icinga\Data\ConfigObject;
class SystemdLogWriter extends LogWriter
{
protected static $severityMap = [
Logger::DEBUG => 7,
Logger::INFO => 6,
Logger::WARNING => 4,
Logger::ERROR => 3,
];
public function __construct()
{
parent::__construct(new ConfigObject([]));
}
public function log($severity, $message)
{
$severity = self::$severityMap[$severity];
echo "<$severity>$message\n";
}
}

51
library/Director/Job/JobRunner.php
View File

@ -1,51 +0,0 @@
<?php
namespace Icinga\Module\Director\Job;
use Icinga\Application\Logger;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Objects\DirectorJob;
class JobRunner
{
public function __construct(Db $db)
{
$this->db = $db;
}
public function runPendingJobs()
{
foreach ($this->getConfiguredJobs() as $job) {
if ($job->shouldRun()) {
Logger::info('Director JobRunner is starting "%s"', $job->job_name);
$this->run($job);
}
}
}
protected function run(DirectorJob $job)
{
if ($this->shouldFork()) {
$this->fork($job);
} else {
$job->run();
}
}
protected function fork(DirectorJob $job)
{
$cmd = 'icingacli director job run ' . $job->id;
$output = `$cmd`;
// TODO: capture output
}
protected function shouldFork()
{
return true;
}
protected function getConfiguredJobs()
{
return DirectorJob::loadAll($this->db);
}
}

6
library/Director/Objects/DirectorJob.php
View File

@ -3,6 +3,7 @@
namespace Icinga\Module\Director\Objects;
use Icinga\Exception\NotFoundError;
use Icinga\Module\Director\Daemon\Logger;
use Icinga\Module\Director\Data\Db\DbObjectWithSettings;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\DirectorObject\Automation\ExportInterface;
@ -79,15 +80,20 @@ class DirectorJob extends DbObjectWithSettings implements ExportInterface
try {
$job->run();
$this->set('last_attempt_succeeded', 'y');
$success = true;
} catch (Exception $e) {
Logger::error($e->getMessage());
$this->set('ts_last_error', date('Y-m-d H:i:s'));
$this->set('last_error_message', $e->getMessage());
$this->set('last_attempt_succeeded', 'n');
$success = false;
}
if ($this->hasBeenModified()) {
$this->store();
}
return $success;
}
/**

85
library/Director/Web/Tabs/MainTabs.php Normal file
View File

@ -0,0 +1,85 @@
<?php
namespace Icinga\Module\Director\Web\Tabs;
use gipfl\Translation\TranslationHelper;
use gipfl\IcingaWeb2\Widget\Tabs;
use Icinga\Authentication\Auth;
use Icinga\Module\Director\Daemon\BackgroundDaemonState;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Health;
use Icinga\Module\Director\Web\Widget\HealthCheckPluginOutput;
class MainTabs extends Tabs
{
use TranslationHelper;
protected $auth;
protected $dbResourceName;
public function __construct(Auth $auth, $dbResourceName)
{
$this->auth = $auth;
$this->dbResourceName = $dbResourceName;
$this->add('main', [
'label' => $this->translate('Overview'),
'url' => 'director'
]);
if ($this->auth->hasPermission('director/admin')) {
$this->add('health', [
'label' => $this->translate('Health'),
'url' => 'director/health'
])->add('daemon', [
'label' => $this->translate('Daemon'),
'url' => 'director/daemon'
]);
}
}
public function render()
{
if ($this->auth->hasPermission('director/admin')) {
if ($this->getActiveName() !== 'health') {
$state = $this->getHealthState();
if ($state->isProblem()) {
$this->get('health')->setTagParams([
'class' => 'state-' . strtolower($state->getName())
]);
}
}
if ($this->getActiveName() !== 'daemon') {
try {
$daemon = new BackgroundDaemonState(Db::fromResourceName($this->dbResourceName));
if ($daemon->isRunning()) {
$state = 'ok';
} else {
$state = 'warning';
}
} catch (\Exception $e) {
$state = 'unknown';
}
if ($state !== 'ok') {
$this->get('daemon')->setTagParams([
'class' => 'state-' . $state
]);
}
}
}
return parent::render();
}
/**
* @return \Icinga\Module\Director\CheckPlugin\PluginState
*/
protected function getHealthState()
{
$health = new Health();
$health->setDbResourceName($this->dbResourceName);
$output = new HealthCheckPluginOutput($health);
return $output->getState();
}
}

130
library/Director/Web/Widget/BackgroundDaemonDetails.php Normal file
View File

@ -0,0 +1,130 @@
<?php
namespace Icinga\Module\Director\Web\Widget;
use gipfl\IcingaWeb2\Icon;
use gipfl\IcingaWeb2\Widget\NameValueTable;
use gipfl\Translation\TranslationHelper;
use Icinga\Date\DateFormatter;
use Icinga\Module\Director\Daemon\RunningDaemonInfo;
use Icinga\Util\Format;
use ipl\Html\BaseHtmlElement;
use ipl\Html\Html;
use ipl\Html\Table;
class BackgroundDaemonDetails extends BaseHtmlElement
{
use TranslationHelper;
protected $tag = 'div';
/** @var RunningDaemonInfo */
protected $info;
/** @var \stdClass TODO: get rid of this */
protected $daemon;
public function __construct(RunningDaemonInfo $info, $daemon)
{
$this->info = $info;
$this->daemon = $daemon;
}
protected function assemble()
{
$info = $this->info;
if ($info->hasBeenStopped()) {
$this->add(Html::tag('p', [
'class' => 'state-hint error'
], Html::sprintf(
$this->translate(
'Daemon has been stopped %s, was running with PID %s as %s@%s'
),
// $info->getHexUuid(),
$this->timeAgo($info->getTimestampStopped() / 1000),
Html::tag('strong', (string) $info->getPid()),
Html::tag('strong', $info->getUsername()),
Html::tag('strong', $info->getFqdn())
)));
} elseif ($info->isOutdated()) {
$this->add(Html::tag('p', [
'class' => 'state-hint error'
], Html::sprintf(
$this->translate(
'Daemon keep-alive is outdated, was last seen running with PID %s as %s@%s %s'
),
// $info->getHexUuid(),
Html::tag('strong', (string) $info->getPid()),
Html::tag('strong', $info->getUsername()),
Html::tag('strong', $info->getFqdn()),
$this->timeAgo($info->getLastUpdate() / 1000)
)));
} else {
$details = new NameValueTable();
$details->addNameValuePairs([
$this->translate('Startup Time') => DateFormatter::formatDateTime($info->getTimestampStarted() / 1000),
$this->translate('PID') => $info->getPid(),
$this->translate('Username') => $info->getUsername(),
$this->translate('FQDN') => $info->getFqdn(),
$this->translate('Running with systemd') => $info->isRunningWithSystemd()
? $this->translate('yes')
: $this->translate('no'),
$this->translate('Binary') => $info->getBinaryPath()
. ($info->binaryRealpathDiffers() ? ' -> ' . $info->getBinaryRealpath() : ''),
$this->translate('PHP Binary') => $info->getPhpBinaryPath()
. ($info->phpBinaryRealpathDiffers() ? ' -> ' . $info->getPhpBinaryRealpath() : ''),
$this->translate('PHP Version') => $info->getPhpVersion(),
$this->translate('PHP Integer') => $info->has64bitIntegers()
? '64bit'
: Html::sprintf(
'%sbit (%s)',
$info->getPhpIntegerSize() * 8,
Html::tag('span', ['class' => 'error'], $this->translate('unsupported'))
),
]);
$this->add($details);
$this->add(Html::tag('p', [
'class' => 'state-hint ok'
], Html::sprintf(
$this->translate(
'Daemon is running with PID %s as %s@%s, last refresh happened %s'
),
// $info->getHexUuid(),
Html::tag('strong', (string)$info->getPid()),
Html::tag('strong', $info->getUsername()),
Html::tag('strong', $info->getFqdn()),
$this->timeAgo($info->getLastUpdate() / 1000)
)));
$this->add(Html::tag('h2', $this->translate('Process List')));
$processes = \json_decode($this->daemon->process_info);
$table = new Table();
$table->add(Html::tag('thead', Html::tag('tr', Html::wrapEach([
'PID',
'Command',
'Memory'
], 'th'))));
$table->setAttribute('class', 'common-table');
foreach ($processes as $pid => $process) {
$table->add($table::row([
[
Icon::create($process->running ? 'ok' : 'warning-empty'),
' ',
$pid
],
Html::tag('pre', $process->command),
Format::bytes($process->memory->rss)
]));
}
$this->add($table);
}
}
protected function timeAgo($time)
{
return Html::tag('span', [
'class' => 'time-ago',
'title' => DateFormatter::formatDateTime($time)
], DateFormatter::timeAgo($time));
}
}

86
library/Director/Web/Widget/Documentation.php Normal file
View File

@ -0,0 +1,86 @@
<?php
namespace Icinga\Module\Director\Web\Widget;
use gipfl\IcingaWeb2\Link;
use gipfl\Translation\TranslationHelper;
use Icinga\Application\ApplicationBootstrap;
use Icinga\Authentication\Auth;
use ipl\Html\Html;
class Documentation
{
use TranslationHelper;
/** @var ApplicationBootstrap */
protected $app;
/** @var Auth */
protected $auth;
public function __construct(ApplicationBootstrap $app, Auth $auth)
{
$this->app = $app;
$this->auth = $auth;
}
public function getModuleLink($label, $module, $chapter, $title = null)
{
if ($title !== null) {
$title = sprintf(
$this->translate('Click to read our documentation: %s'),
$title
);
}
$linkToGitHub = false;
$hasModule = $this->app->getModuleManager()->hasLoaded($module);
if ($hasModule && $this->hasAccessToDocumentationModule()) {
return Link::create(
$label,
'doc/module/director/chapter/' . \preg_replace('/^\d+-/', '', \rawurlencode($chapter)),
null,
[
'data-base-target' => '_next',
'class' => 'icon-book',
'title' => $title,
]
);
} elseif ($linkToGitHub) {
return Html::tag('a', [
'href' => $this->githubDocumentationUrl($module, $chapter),
'target' => '_blank',
'title' => $title,
], $label);
} else {
return Html::tag('a', [
'href' => $this->icingaDocumentationUrl($module, $chapter),
'target' => '_blank',
'title' => $title,
], $label);
}
}
protected function githubDocumentationUrl($module, $chapter)
{
return sprintf(
"https://github.com/Icinga/icingaweb2-module-%s/blob/master/doc/%s.md",
\rawurlencode($module),
\rawurlencode($chapter)
);
}
protected function icingaDocumentationUrl($module, $chapter)
{
return sprintf(
'https://icinga.com/docs/%s/latest/doc/%s/',
\rawurlencode($module),
\rawurlencode($chapter)
);
}
protected function hasAccessToDocumentationModule()
{
return $this->app->getModuleManager()->hasLoaded('doc')
&& $this->auth->hasPermission('module/doc');
}
}

13
library/Director/Web/Widget/ImportSourceDetails.php
View File

@ -20,9 +20,6 @@ class ImportSourceDetails extends HtmlDocument
$this->source = $source;
}
/**
* @throws \Icinga\Exception\IcingaException
*/
protected function assemble()
{
$source = $this->source;
@ -35,7 +32,7 @@ class ImportSourceDetails extends HtmlDocument
case 'unknown':
$this->add(Html::tag(
'p',
null,
['class' => 'state-hint warning'],
$this->translate(
"It's currently unknown whether we are in sync with this Import Source."
. ' You should either check for changes or trigger a new Import Run.'
@ -43,7 +40,7 @@ class ImportSourceDetails extends HtmlDocument
));
break;
case 'in-sync':
$this->add(Html::tag('p', null, sprintf(
$this->add(Html::tag('p', ['class' => 'state-hint ok'], sprintf(
$this->translate(
'This Import Source was last found to be in sync at %s.'
),
@ -54,13 +51,13 @@ class ImportSourceDetails extends HtmlDocument
// - there have been activities since then
break;
case 'pending-changes':
$this->add(Html::tag('p', ['class' => 'warning'], $this->translate(
$this->add(Html::tag('p', ['class' => 'state-hint warning'], $this->translate(
'There are pending changes for this Import Source. You should trigger a new'
. ' Import Run.'
)));
break;
case 'failing':
$this->add(Html::tag('p', ['class' => 'error'], sprintf(
$this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf(
$this->translate(
'This Import Source failed when last checked at %s: %s'
),
@ -69,7 +66,7 @@ class ImportSourceDetails extends HtmlDocument
)));
break;
default:
$this->add(Html::tag('p', ['class' => 'error'], sprintf(
$this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf(
$this->translate('This Import Source has an invalid state: %s'),
$source->get('import_state')
)));

40
library/Director/Web/Widget/JobDetails.php
View File

@ -2,6 +2,7 @@
namespace Icinga\Module\Director\Web\Widget;
use Icinga\Date\DateFormatter;
use ipl\Html\HtmlDocument;
use Icinga\Module\Director\Objects\DirectorJob;
use ipl\Html\Html;
@ -18,43 +19,52 @@ class JobDetails extends HtmlDocument
*/
public function __construct(DirectorJob $job)
{
if ($job->disabled === 'y') {
$this->add(Html::tag('p', ['class' => 'error'], sprintf(
$runInterval = $job->get('run_interval');
if ($job->hasBeenDisabled()) {
$this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf(
$this->translate(
'This job would run every %ds. It has been disabled and will'
. ' therefore not be executed as scheduled'
),
$job->run_interval
$runInterval
)));
} else {
//$class = $job->job(); echo $class::getDescription()
$msg = $job->isPending()
? sprintf(
$this->translate('This job runs every %ds and is currently pending'),
$job->run_interval
$runInterval
)
: sprintf(
$this->translate('This job runs every %ds.'),
$job->run_interval
$runInterval
);
$this->add(Html::tag('p', null, $msg));
}
if ($job->ts_last_attempt) {
if ($job->last_attempt_succeeded) {
$this->add(Html::tag('p', null, sprintf(
$this->translate('The last attempt succeeded at %s'),
$job->ts_last_attempt
$tsLastAttempt = $job->get('ts_last_attempt');
$ts = \strtotime($tsLastAttempt);
$timeAgo = Html::tag('span', [
'class' => 'time-ago',
'title' => DateFormatter::formatDateTime($ts)
], DateFormatter::timeAgo($ts));
if ($tsLastAttempt) {
if ($job->get('last_attempt_succeeded') === 'y') {
$this->add(Html::tag('p', ['class' => 'state-hint ok'], Html::sprintf(
$this->translate('The last attempt succeeded %s'),
$timeAgo
)));
} else {
$this->add(Html::tag('p', ['class' => 'error'], sprintf(
$this->translate('The last attempt failed at %s: %s'),
$job->ts_last_attempt,
$job->ts_last_error
$this->add(Html::tag('p', ['class' => 'state-hint error'], Html::sprintf(
$this->translate('The last attempt failed %s: %s'),
$timeAgo,
$job->get('last_error_message')
)));
}
} else {
$this->add(Html::tag('p', null, $this->translate('This job has not been executed yet')));
$this->add(Html::tag('p', [
'class' => 'state-hint warning'
], $this->translate('This job has not been executed yet')));
}
}
}

2
module.info
View File

@ -1,6 +1,6 @@
Name: Icinga Director
Version: master
Depends: reactbundle (>=0.6.0), ipl (>=0.3.0), incubator (>=0.3.0)
Depends: reactbundle (>=0.7.0), ipl (>=0.3.0), incubator (>=0.4.0)
Description: Director - Config tool for Icinga 2
Icinga Director is a configuration tool that has been designed to make
Icinga 2 configuration easy and understandable.

49
public/css/module.less
View File

@ -7,6 +7,10 @@ div.action-bar a:focus, .tabs a:focus {
}
}
a:before {
text-decoration: none;
}
form:focus {
outline: none;
}
@ -1075,7 +1079,7 @@ span.error {
}
}
p.error {
p.error:not(.state-hint) {
color: white;
padding: 1em 2em;
background-color: @colorCritical;
@ -1087,7 +1091,7 @@ p.error {
}
}
p.warning {
p.warning:not(.state-hint) {
color: white;
padding: 1em 2em;
background-color: @colorWarning;
@ -1099,7 +1103,7 @@ p.warning {
}
}
p.information {
p.information:not(.state-hint) {
color: white;
padding: 1em 2em;
background-color: @colorOk;
@ -1111,6 +1115,45 @@ p.information {
}
}
p.state-hint {
border: 1px solid @text-color;
padding: 0.5em;
line-height: 2em;
max-width: 60em;
border-left-width: 3em;
&:before {
position: relative;
margin-left: -1.5em;
margin-right: 0.5em;
height: 100%;
vertical-align: middle;
font-family: 'ifont';
color: white;
font-size: 2em;
}
&.ok {
border-color: @color-ok;
}
&.warning {
border-color: @color-warning;
}
&.error {
border-color: @color-critical;
}
&.critical:before, &.error:before {
content: '\e885';
}
&.warning:before {
content: '\e885';
}
&.ok:before {
content: '\e803';
}
a {
text-decoration: underline;
}
}
table th.actions, table td.actions {
text-align: right;
}

25
schema/mysql-migrations/upgrade_167.sql Normal file
View File

@ -0,0 +1,25 @@
CREATE TABLE director_daemon_info (
instance_uuid_hex VARCHAR(32) NOT NULL, -- random by daemon
schema_version SMALLINT UNSIGNED NOT NULL,
fqdn VARCHAR(255) NOT NULL,
username VARCHAR(64) NOT NULL,
pid INT UNSIGNED NOT NULL,
binary_path VARCHAR(128) NOT NULL,
binary_realpath VARCHAR(128) NOT NULL,
php_binary_path VARCHAR(128) NOT NULL,
php_binary_realpath VARCHAR(128) NOT NULL,
php_version VARCHAR(64) NOT NULL,
php_integer_size SMALLINT NOT NULL,
running_with_systemd ENUM('y', 'n') NOT NULL,
ts_started BIGINT(20) NOT NULL,
ts_stopped BIGINT(20) DEFAULT NULL,
ts_last_modification BIGINT(20) DEFAULT NULL,
ts_last_update BIGINT(20) DEFAULT NULL,
process_info MEDIUMTEXT NOT NULL,
PRIMARY KEY (instance_uuid_hex)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_bin;
INSERT INTO director_schema_migration
(schema_version, migration_time)
VALUES (167, NOW());

23
schema/mysql.sql
View File

@ -9,6 +9,27 @@
SET sql_mode = 'STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,ANSI_QUOTES,ERROR_FOR_DIVISION_BY_ZERO';
CREATE TABLE director_daemon_info (
instance_uuid_hex VARCHAR(32) NOT NULL, -- random by daemon
schema_version SMALLINT UNSIGNED NOT NULL,
fqdn VARCHAR(255) NOT NULL,
username VARCHAR(64) NOT NULL,
pid INT UNSIGNED NOT NULL,
binary_path VARCHAR(128) NOT NULL,
binary_realpath VARCHAR(128) NOT NULL,
php_binary_path VARCHAR(128) NOT NULL,
php_binary_realpath VARCHAR(128) NOT NULL,
php_version VARCHAR(64) NOT NULL,
php_integer_size SMALLINT NOT NULL,
running_with_systemd ENUM('y', 'n') NOT NULL,
ts_started BIGINT(20) NOT NULL,
ts_stopped BIGINT(20) DEFAULT NULL,
ts_last_modification BIGINT(20) DEFAULT NULL,
ts_last_update BIGINT(20) DEFAULT NULL,
process_info MEDIUMTEXT NOT NULL,
PRIMARY KEY (instance_uuid_hex)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_bin;
CREATE TABLE director_activity_log (
id BIGINT(20) UNSIGNED AUTO_INCREMENT NOT NULL,
object_type VARCHAR(64) NOT NULL,
@ -1848,4 +1869,4 @@ CREATE TABLE icinga_scheduled_downtime_range (
INSERT INTO director_schema_migration
(schema_version, migration_time)
VALUES (165, NOW());
VALUES (167, NOW());

24
schema/pgsql-migrations/upgrade_167.sql Normal file
View File

@ -0,0 +1,24 @@
CREATE TABLE director_daemon_info (
instance_uuid_hex character varying(32) NOT NULL, -- random by daemon
schema_version SMALLINT NOT NULL,
fqdn character varying(255) NOT NULL,
username character varying(64) NOT NULL,
pid integer NOT NULL,
binary_path character varying(128) NOT NULL,
binary_realpath character varying(128) NOT NULL,
php_binary_path character varying(128) NOT NULL,
php_binary_realpath character varying(128) NOT NULL,
php_version character varying(64) NOT NULL,
php_integer_size SMALLINT NOT NULL,
running_with_systemd enum_boolean DEFAULT NULL,
ts_started bigint NOT NULL,
ts_stopped bigint DEFAULT NULL,
ts_last_modification bigint DEFAULT NULL,
ts_last_update bigint NOT NULL,
process_info text NOT NULL,
PRIMARY KEY (instance_uuid_hex)
);
INSERT INTO director_schema_migration
(schema_version, migration_time)
VALUES (167, NOW());

24
schema/pgsql.sql
View File

@ -53,6 +53,28 @@ CREATE OR REPLACE FUNCTION unix_timestamp(timestamp with time zone) RETURNS bigi
' LANGUAGE sql;
CREATE TABLE director_daemon_info (
instance_uuid_hex character varying(32) NOT NULL, -- random by daemon
schema_version SMALLINT NOT NULL,
fqdn character varying(255) NOT NULL,
username character varying(64) NOT NULL,
pid integer NOT NULL,
binary_path character varying(128) NOT NULL,
binary_realpath character varying(128) NOT NULL,
php_binary_path character varying(128) NOT NULL,
php_binary_realpath character varying(128) NOT NULL,
php_version character varying(64) NOT NULL,
php_integer_size SMALLINT NOT NULL,
running_with_systemd enum_boolean DEFAULT NULL,
ts_started bigint NOT NULL,
ts_stopped bigint DEFAULT NULL,
ts_last_modification bigint DEFAULT NULL,
ts_last_update bigint NOT NULL,
process_info text NOT NULL,
PRIMARY KEY (instance_uuid_hex)
);
CREATE TABLE director_activity_log (
id bigserial,
object_type character varying(64) NOT NULL,
@ -2160,4 +2182,4 @@ COMMENT ON COLUMN icinga_scheduled_downtime_range.merge_behaviour IS 'set -> = {
INSERT INTO director_schema_migration
(schema_version, migration_time)
VALUES (165, NOW());
VALUES (167, NOW());