From 4ee3ef2fd9e34bfaa17cc5562bbd2ce7f5af5c1a Mon Sep 17 00:00:00 2001 From: Thomas Gelf Date: Tue, 17 Sep 2019 15:06:39 +0200 Subject: [PATCH] Daemon: new implementation --- application/clicommands/DaemonCommand.php | 26 ++ application/clicommands/JobsCommand.php | 114 +++---- application/controllers/DaemonController.php | 59 ++++ .../controllers/DashboardController.php | 30 +- application/controllers/HealthController.php | 10 +- .../controllers/ImportsourceController.php | 9 +- application/controllers/JobController.php | 1 + .../controllers/SyncruleController.php | 8 +- contrib/systemd/icinga-director.service | 21 ++ doc/05-Upgrading.md | 5 +- doc/60-CLI.md | 17 - doc/75-Background-Daemon.md | 68 ++++ library/Director/Daemon/BackgroundDaemon.php | 212 ++++++++++++ .../Director/Daemon/BackgroundDaemonState.php | 61 ++++ library/Director/Daemon/DaemonDb.php | 317 ++++++++++++++++++ .../Director/Daemon/DaemonProcessDetails.php | 122 +++++++ .../Director/Daemon/DaemonProcessState.php | 85 +++++ library/Director/Daemon/DaemonUtil.php | 16 + library/Director/Daemon/DbBasedComponent.php | 19 ++ library/Director/Daemon/JobRunner.php | 234 +++++++++++++ library/Director/Daemon/JsonRpcLogWriter.php | 37 ++ library/Director/Daemon/LogProxy.php | 76 +++++ library/Director/Daemon/Logger.php | 21 ++ library/Director/Daemon/ProcessList.php | 125 +++++++ library/Director/Daemon/RunningDaemonInfo.php | 154 +++++++++ library/Director/Daemon/SystemdLogWriter.php | 27 ++ library/Director/Job/JobRunner.php | 51 --- library/Director/Objects/DirectorJob.php | 6 + library/Director/Web/Tabs/MainTabs.php | 85 +++++ .../Web/Widget/BackgroundDaemonDetails.php | 130 +++++++ library/Director/Web/Widget/Documentation.php | 86 +++++ .../Web/Widget/ImportSourceDetails.php | 13 +- library/Director/Web/Widget/JobDetails.php | 40 ++- module.info | 2 +- public/css/module.less | 49 ++- 35 files changed, 2136 insertions(+), 200 deletions(-) create mode 100644 application/clicommands/DaemonCommand.php create mode 100644 application/controllers/DaemonController.php create mode 100644 contrib/systemd/icinga-director.service create mode 100644 doc/75-Background-Daemon.md create mode 100644 library/Director/Daemon/BackgroundDaemon.php create mode 100644 library/Director/Daemon/BackgroundDaemonState.php create mode 100644 library/Director/Daemon/DaemonDb.php create mode 100644 library/Director/Daemon/DaemonProcessDetails.php create mode 100644 library/Director/Daemon/DaemonProcessState.php create mode 100644 library/Director/Daemon/DaemonUtil.php create mode 100644 library/Director/Daemon/DbBasedComponent.php create mode 100644 library/Director/Daemon/JobRunner.php create mode 100644 library/Director/Daemon/JsonRpcLogWriter.php create mode 100644 library/Director/Daemon/LogProxy.php create mode 100644 library/Director/Daemon/Logger.php create mode 100644 library/Director/Daemon/ProcessList.php create mode 100644 library/Director/Daemon/RunningDaemonInfo.php create mode 100644 library/Director/Daemon/SystemdLogWriter.php delete mode 100644 library/Director/Job/JobRunner.php create mode 100644 library/Director/Web/Tabs/MainTabs.php create mode 100644 library/Director/Web/Widget/BackgroundDaemonDetails.php create mode 100644 library/Director/Web/Widget/Documentation.php diff --git a/application/clicommands/DaemonCommand.php b/application/clicommands/DaemonCommand.php new file mode 100644 index 00000000..e89e1da8 --- /dev/null +++ b/application/clicommands/DaemonCommand.php @@ -0,0 +1,26 @@ +] + */ + public function runAction() + { + $this->app->getModuleManager()->loadEnabledModules(); + $daemon = new BackgroundDaemon(); + if ($dbResource = $this->params->get('db-resource')) { + $daemon->setDbResourceName($dbResource); + } + $daemon->run(); + } +} diff --git a/application/clicommands/JobsCommand.php b/application/clicommands/JobsCommand.php index adddd745..3f12adeb 100644 --- a/application/clicommands/JobsCommand.php +++ b/application/clicommands/JobsCommand.php @@ -2,78 +2,72 @@ namespace Icinga\Module\Director\Clicommands; -use Icinga\Module\Director\Cli\Command; -use Icinga\Module\Director\Job\JobRunner; -use Icinga\Module\Director\Objects\DirectorJob; -use Icinga\Application\Logger; use Exception; +use gipfl\Cli\Process; +use gipfl\Protocol\JsonRpc\Connection; +use gipfl\Protocol\NetString\StreamWrapper; +use Icinga\Module\Director\Cli\Command; +use Icinga\Module\Director\Daemon\JsonRpcLogWriter as JsonRpcLogWriterAlias; +use Icinga\Module\Director\Daemon\Logger; +use Icinga\Module\Director\Objects\DirectorJob; +use React\EventLoop\Factory as Loop; +use React\EventLoop\LoopInterface; +use React\Stream\ReadableResourceStream; +use React\Stream\WritableResourceStream; class JobsCommand extends Command { public function runAction() { - $forever = $this->params->shift('forever'); - if (! $forever && $this->params->getStandalone() === 'forever') { - $forever = true; - $this->params->shift(); + $loop = Loop::create(); + if ($this->params->get('rpc')) { + $this->enableRpc($loop); } - - $jobId = $this->params->shift(); - if ($jobId) { - $this->raiseLimits(); - $job = DirectorJob::loadWithAutoIncId($jobId, $this->db()); - $job->run(); - exit(0); - } - - if ($forever) { - $this->runforever(); + if ($this->params->get('rpc') && $jobId = $this->params->get('id')) { + $exitCode = 1; + $jobId = (int) $jobId; + $loop->futureTick(function () use ($jobId, $loop, &$exitCode) { + Process::setTitle('icinga::director::job'); + try { + $this->raiseLimits(); + $job = DirectorJob::loadWithAutoIncId($jobId, $this->db()); + Process::setTitle('icinga::director::job (' . $job->get('job_name') . ')'); + if ($job->run()) { + $exitCode = 0; + } else { + $exitCode = 1; + } + } catch (Exception $e) { + Logger::error($e->getMessage()); + $exitCode = 1; + } + $loop->futureTick(function () use ($loop) { + $loop->stop(); + }); + }); } else { - $this->runAllPendingJobs(); + Logger::error('This command is no longer available. Please check our Upgrading documentation'); + $exitCode = 1; } + + $loop->run(); + exit($exitCode); } - protected function runforever() + protected function enableRpc(LoopInterface $loop) { - // We'll terminate ourselves after 24h for now: - $runUnless = time() + 86400; + // stream_set_blocking(STDIN, 0); + // stream_set_blocking(STDOUT, 0); + // print_r(stream_get_meta_data(STDIN)); + // stream_set_write_buffer(STDOUT, 0); + // ini_set('implicit_flush', 1); + $netString = new StreamWrapper( + new ReadableResourceStream(STDIN, $loop), + new WritableResourceStream(STDOUT, $loop) + ); + $jsonRpc = new Connection(); + $jsonRpc->handle($netString); - // We'll exit in case more than 100MB of memory are still in use - // after the last job execution: - $maxMem = 100 * 1024 * 1024; - - while (true) { - $this->runAllPendingJobs(); - if (memory_get_usage() > $maxMem) { - exit(0); - } - - if (time() > $runUnless) { - exit(0); - } - - sleep(2); - } - } - - protected function runAllPendingJobs() - { - $jobs = new JobRunner($this->db()); - - try { - if ($this->hasBeenDisabled()) { - return; - } - - $jobs->runPendingJobs(); - } catch (Exception $e) { - Logger::error('Director Job Error: ' . $e->getMessage()); - sleep(10); - } - } - - protected function hasBeenDisabled() - { - return $this->db()->settings()->disable_all_jobs === 'y'; + Logger::replaceRunningInstance(new JsonRpcLogWriterAlias($jsonRpc)); } } diff --git a/application/controllers/DaemonController.php b/application/controllers/DaemonController.php new file mode 100644 index 00000000..3d17c682 --- /dev/null +++ b/application/controllers/DaemonController.php @@ -0,0 +1,59 @@ +setAutorefreshInterval(10); + $this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('daemon'); + $this->setTitle($this->translate('Director Background Daemon')); + // Avoiding layout issues: + $this->content()->add(Html::tag('h1', $this->translate('Director Background Daemon'))); + // TODO: move dashboard titles into controls. Or figure out whether 2.7 "broke" this + + $error = null; + try { + $db = $this->db()->getDbAdapter(); + $daemons = $db->fetchAll( + $db->select()->from('director_daemon_info')->order('fqdn')->order('username')->order('pid') + ); + } catch (\Exception $e) { + $daemons = []; + $error = $e->getMessage(); + } + + if (empty($daemons)) { + $documentation = new Documentation(Icinga::app(), $this->Auth()); + $message = Html::sprintf($this->translate( + 'The Icinga Director Background Daemon is not running.' + . ' Please check our %s in case you need step by step instructions' + . ' showing you how to fix this.' + ), $documentation->getModuleLink( + $this->translate('documentation'), + 'director', + '75-Background-Daemon', + $this->translate('Icinga Director Background Daemon') + )); + $this->content()->add(Html::tag('p', ['class' => 'state-hint error'], [ + $message, + ($error ? [Html::tag('br'), Html::tag('strong', $error)] : ''), + ])); + return; + } + + foreach ($daemons as $daemon) { + $info = new RunningDaemonInfo($daemon); + $this->content()->add([new BackgroundDaemonDetails($info, $daemon) /*, $logWindow*/]); + } + } +} diff --git a/application/controllers/DashboardController.php b/application/controllers/DashboardController.php index 38d84ec2..58fe8027 100644 --- a/application/controllers/DashboardController.php +++ b/application/controllers/DashboardController.php @@ -2,6 +2,7 @@ namespace Icinga\Module\Director\Controllers; +use Icinga\Module\Director\Web\Tabs\MainTabs; use Icinga\Module\Director\Web\Widget\HealthCheckPluginOutput; use Icinga\Module\Director\Dashboard\Dashboard; use Icinga\Module\Director\Health; @@ -45,22 +46,7 @@ class DashboardController extends ActionController $dashboard = Dashboard::loadByName($name, $this->db()); $this->tabs($dashboard->getTabs())->activate($name); } else { - $this->tabs()->add('main', [ - 'label' => $this->translate('Overview'), - 'url' => 'director' - ])->activate('main'); - if ($this->hasPermission('director/admin')) { - $this->tabs()->add('health', [ - 'label' => $this->translate('Health'), - 'url' => 'director/health' - ]); - $state = $this->getHealthState(); - if ($state->isProblem()) { - $this->tabs()->get('health')->setTagParams([ - 'class' => 'state-' . strtolower($state->getName()) - ]); - } - } + $this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('main'); } $cntDashboards = 0; @@ -83,16 +69,4 @@ class DashboardController extends ActionController $this->content()->add($msg); } } - - /** - * @return \Icinga\Module\Director\CheckPlugin\PluginState - */ - protected function getHealthState() - { - $health = new Health(); - $health->setDbResourceName($this->getDbResourceName()); - $output = new HealthCheckPluginOutput($health); - - return $output->getState(); - } } diff --git a/application/controllers/HealthController.php b/application/controllers/HealthController.php index 87c30606..4fac4d26 100644 --- a/application/controllers/HealthController.php +++ b/application/controllers/HealthController.php @@ -2,6 +2,7 @@ namespace Icinga\Module\Director\Controllers; +use Icinga\Module\Director\Web\Tabs\MainTabs; use ipl\Html\Html; use Icinga\Module\Director\Web\Widget\HealthCheckPluginOutput; use Icinga\Module\Director\Health; @@ -12,14 +13,7 @@ class HealthController extends ActionController public function indexAction() { $this->setAutorefreshInterval(10); - $this->tabs()->add('main', [ - 'label' => $this->translate('Overview'), - 'url' => 'director' - ])->add('health', [ - 'label' => $this->translate('Health'), - 'url' => 'director/health' - ])->activate('health'); - + $this->tabs(new MainTabs($this->Auth(), $this->getDbResourceName()))->activate('health'); $this->setTitle($this->translate('Director Health')); $health = new Health(); $health->setDbResourceName($this->getDbResourceName()); diff --git a/application/controllers/ImportsourceController.php b/application/controllers/ImportsourceController.php index f51524f5..613ad95a 100644 --- a/application/controllers/ImportsourceController.php +++ b/application/controllers/ImportsourceController.php @@ -2,7 +2,6 @@ namespace Icinga\Module\Director\Controllers; -use Icinga\Exception\NotFoundError; use Icinga\Module\Director\Forms\ImportRowModifierForm; use Icinga\Module\Director\Forms\ImportSourceForm; use Icinga\Module\Director\Web\ActionBar\AutomationObjectActionBar; @@ -44,6 +43,9 @@ class ImportsourceController extends ActionController } } + /** + * @throws \Icinga\Exception\NotFoundError + */ protected function addMainActions() { $this->actions(new AutomationObjectActionBar( @@ -66,7 +68,6 @@ class ImportsourceController extends ActionController } /** - * @throws \Icinga\Exception\IcingaException * @throws \Icinga\Exception\NotFoundError */ public function indexAction() @@ -95,7 +96,7 @@ class ImportsourceController extends ActionController } /** - * @throws NotFoundError + * @throws \Icinga\Exception\NotFoundError */ public function editAction() { @@ -239,7 +240,7 @@ class ImportsourceController extends ActionController /** * @return ImportSource - * @throws NotFoundError + * @throws \Icinga\Exception\NotFoundError */ protected function getImportSource() { diff --git a/application/controllers/JobController.php b/application/controllers/JobController.php index 3164ee55..04f9e0ad 100644 --- a/application/controllers/JobController.php +++ b/application/controllers/JobController.php @@ -16,6 +16,7 @@ class JobController extends ActionController */ public function indexAction() { + $this->setAutorefreshInterval(10); $job = $this->requireJob(); $this ->addJobTabs($job, 'show') diff --git a/application/controllers/SyncruleController.php b/application/controllers/SyncruleController.php index ec439f14..7b2ba6c3 100644 --- a/application/controllers/SyncruleController.php +++ b/application/controllers/SyncruleController.php @@ -130,7 +130,7 @@ class SyncruleController extends ActionController */ protected function warning($msg) { - $this->content()->add(Html::tag('p', ['class' => 'warning'], $msg)); + $this->content()->add(Html::tag('p', ['class' => 'state-hint warning'], $msg)); } /** @@ -138,7 +138,7 @@ class SyncruleController extends ActionController */ protected function error($msg) { - $this->content()->add(Html::tag('p', ['class' => 'error'], $msg)); + $this->content()->add(Html::tag('p', ['class' => 'state-hint error'], $msg)); } /** @@ -165,7 +165,7 @@ class SyncruleController extends ActionController } catch (\Exception $e) { $this->content()->add( Html::tag('p', [ - 'class' => 'error' + 'class' => 'state-hint error' ], $e->getMessage()) ); @@ -174,7 +174,7 @@ class SyncruleController extends ActionController if (empty($modifications)) { $this->content()->add(Html::tag('p', [ - 'class' => 'information' + 'class' => 'state-hint ok' ], $this->translate('This Sync Rule is in sync and would currently not apply any changes'))); return; diff --git a/contrib/systemd/icinga-director.service b/contrib/systemd/icinga-director.service new file mode 100644 index 00000000..f96f1d71 --- /dev/null +++ b/contrib/systemd/icinga-director.service @@ -0,0 +1,21 @@ +[Unit] +Description=Icinga Director - Monitoring Configuration +Documentation=https://icinga.com/docs/director/latest/ +Wants=network.target + +[Service] +EnvironmentFile=-/etc/default/icinga-director +EnvironmentFile=-/etc/sysconfig/icinga-director +ExecStart=/usr/bin/icingacli director daemon run +ExecReload=/bin/kill -HUP ${MAINPID} +User=icingadirector +SyslogIdentifier=icingadirector +Type=notify + +NotifyAccess=main +WatchdogSec=10 +RestartSec=30 +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/doc/05-Upgrading.md b/doc/05-Upgrading.md index 57bddfcc..ea5b73f1 100644 --- a/doc/05-Upgrading.md +++ b/doc/05-Upgrading.md @@ -54,7 +54,10 @@ Apart from this, in case you are running 1.6.x or any GIT master since then, all you need is to replace the Director module folder with the new one. Or to run `git checkout v1.7.x` in case you installed Director from GIT. -As always, you'll then be prompted to apply pending Database Migrations. +As always, you'll then be prompted to apply pending Database Migrations. There +is now a new, modern (and mandatory) Background Daemon, the old (optional) Jobs +Daemon must be removed. Please check our [documentation](75-Background-Daemon.md) +for related instructions. Upgrading to 1.6.x ------------------------------------------------- diff --git a/doc/60-CLI.md b/doc/60-CLI.md index 7e0ce0f7..c72da7a3 100644 --- a/doc/60-CLI.md +++ b/doc/60-CLI.md @@ -593,23 +593,6 @@ with existing ones and persists eventual changes. | `--id ` | A Sync Rule ID. Use the list command to figure out | | `--benchmark` | Show timing and memory usage details | -### Running Jobs -The `jobs` command runs pending Import and Sync jobs. Please note that we have -planned a scheduler configurable through the Icinga Director web interface, but -this is not available yes. - -So the only option you have right now is to trigger all jobs at once: - -```shell -icingacli director jobs run -``` - -The output could look as follows: - -``` -Import "Puppet DB (PE 2015)" provides changes, triggering run... SUCCEEDED -Sync rule "Hosts from PE2015" provides changes, triggering sync... SUCCEEDED -``` Database housekeeping --------------------- diff --git a/doc/75-Background-Daemon.md b/doc/75-Background-Daemon.md new file mode 100644 index 00000000..47c89cb6 --- /dev/null +++ b/doc/75-Background-Daemon.md @@ -0,0 +1,68 @@ +Background-Daemon +=============================================== + +The Icinga Director Background Daemon is available (and mandatory) since v1.7.0. +It is responsible for various background tasks, including fully automated Import, +Sync & Config Deployment Tasks. + +Daemon Installation +------------------- + +In case you installed Icinga Director as a package, the daemon should already +have been installed. In case you're running directly from a GIT working copy or +from a manual installation, you need to tell `systemd` about your new service. + +First make sure that the system user `icingadirector` exists. In case it doesn't, +please create one: + +```sh +useradd -r -g icingaweb2 -d /var/lib/icingadirector -s /bin/false icingadirector +install -d -o icingadirector -g icingaweb2 -m 0750 /var/lib/icingadirector +``` + +Then copy the provided Unit-File from our [contrib](../contrib/systemd/icinga-director.service) +to `/etc/systemd/system`, enable and start the service: + +```sh +MODULE_PATH=/usr/share/icingaweb2/modules/director +cp "${MODULE_PATH}/contrib/systemd/icinga-director.service" /etc/systemd/system/ +systemctl daemon-reload +``` + +Now your system knows about the Icinga Director Daemon. You should make sure that +it starts automatically each time your system boots: + +```sh +systemctl enable icinga-director.service +``` + +Starting the Daemon +------------------- + +You now can start the Background daemon like any other service on your Linux system: + +```sh +systemctl enable icinga-director.service +``` + +Stopping the Daemon +------------------- + +You now can start the Background daemon like any other service on your Linux system: + +```sh +systemctl enable icinga-director.service +``` + +Getting rid of the old Job Daemon +--------------------------------- + +Before v1.7.0, Icinga Director shipped an optional Job Daemon. This one is no longer +needed and should be removed from your system as follows: + +```sh +systemctl stop director-jobs +systemctl disable director-jobs +rm /etc/systemd/system/director-jobs.service +systemctl daemon-reload +``` \ No newline at end of file diff --git a/library/Director/Daemon/BackgroundDaemon.php b/library/Director/Daemon/BackgroundDaemon.php new file mode 100644 index 00000000..c4cfd7f5 --- /dev/null +++ b/library/Director/Daemon/BackgroundDaemon.php @@ -0,0 +1,212 @@ +loop = $loop; + $this->loop->futureTick(function () { + $this->initialize(); + }); + if ($ownLoop) { + $loop->run(); + } + } + + public function setDbResourceName($name) + { + $this->dbResourceName = $name; + + return $this; + } + + protected function initialize() + { + $this->registerSignalHandlers($this->loop); + $this->processState = new DaemonProcessState('icinga::director'); + $this->jobRunner = new JobRunner($this->loop); + $this->systemd = $this->eventuallyInitializeSystemd(); + $this->processState->setSystemd($this->systemd); + $this->processDetails = $this + ->initializeProcessDetails($this->systemd) + ->registerProcessList($this->jobRunner->getProcessList()); + $this->logProxy = new LogProxy($this->processDetails->getInstanceUuid()); + $this->jobRunner->forwardLog($this->logProxy); + $this->daemonDb = $this->initializeDb( + $this->processDetails, + $this->processState, + $this->dbResourceName + ); + $this->daemonDb + ->register($this->jobRunner) + ->register($this->logProxy) + ->run($this->loop); + if ($this->systemd) { + $this->systemd->setReady(); + } + } + + /** + * @param NotifySystemD|false $systemd + * @return DaemonProcessDetails + */ + protected function initializeProcessDetails($systemd) + { + if ($systemd && $systemd->hasInvocationId()) { + $uuid = $systemd->getInvocationId(); + } else { + try { + $uuid = \bin2hex(Uuid::uuid4()->getBytes()); + } catch (Exception $e) { + $uuid = 'deadc0de' . \substr(\md5(\getmypid()), 0, 24); + } + } + $processDetails = new DaemonProcessDetails($uuid); + if ($systemd) { + $processDetails->set('running_with_systemd', 'y'); + } + + return $processDetails; + } + + protected function eventuallyInitializeSystemd() + { + $systemd = NotifySystemD::ifRequired($this->loop); + if ($systemd) { + Logger::replaceRunningInstance(new SystemdLogWriter()); + Logger::info(sprintf( + "Started by systemd, notifying watchdog every %0.2Gs via %s", + $systemd->getWatchdogInterval(), + $systemd->getSocketPath() + )); + } else { + Logger::debug('Running without systemd'); + } + + return $systemd; + } + + protected function initializeDb( + DaemonProcessDetails $processDetails, + DaemonProcessState $processState, + $dbResourceName = null + ) { + $db = new DaemonDb($processDetails); + $db->on('state', function ($state) use ($processState) { + $processState->setComponentState('db', $state); + }); + + $db->setConfigWatch( + $dbResourceName + ? DbResourceConfigWatch::name($dbResourceName) + : DbResourceConfigWatch::module('director') + ); + + return $db; + } + + protected function registerSignalHandlers(LoopInterface $loop) + { + $func = function ($signal) use (&$func) { + $this->shutdownWithSignal($signal, $func); + }; + $funcReload = function () { + $this->reload(); + }; + $loop->addSignal(SIGHUP, $funcReload); + $loop->addSignal(SIGINT, $func); + $loop->addSignal(SIGTERM, $func); + } + + protected function shutdownWithSignal($signal, &$func) + { + $this->loop->removeSignal($signal, $func); + $this->shutdown(); + } + + protected function reload() + { + if ($this->reloading) { + Logger::error('Ignoring reload request, reload is already in progress'); + return; + } + $this->reloading = true; + $this->setState('reloading the main process'); + $this->daemonDb->disconnect()->then(function () { + Process::restart(); + }); + } + + protected function shutdown() + { + if ($this->shuttingDown) { + Logger::error('Ignoring shutdown request, shutdown is already in progress'); + return; + } + Logger::info('Shutting down'); + $this->shuttingDown = true; + $this->setState('shutting down'); + $this->daemonDb->disconnect()->then(function () { + Logger::info('DB has been disconnected, shutdown finished'); + $this->loop->stop(); + }); + } + + protected function setState($state) + { + if ($this->processState) { + $this->processState->setState($state); + } + + return $this; + } +} diff --git a/library/Director/Daemon/BackgroundDaemonState.php b/library/Director/Daemon/BackgroundDaemonState.php new file mode 100644 index 00000000..beb10b92 --- /dev/null +++ b/library/Director/Daemon/BackgroundDaemonState.php @@ -0,0 +1,61 @@ +db = $db; + } + + public function hasProblems() + { + return $this->isRunning(); + } + + public function isRunning() + { + foreach ($this->getInstances() as $instance) { + if ($instance->isRunning()) { + return true; + } + } + + return false; + } + + public function getInstances() + { + if ($this->instances === null) { + $this->instances = $this->fetchInfo(); + } + + return $this->instances; + } + + /** + * @return RunningDaemonInfo[] + */ + protected function fetchInfo() + { + $db = $this->db->getDbAdapter(); + $daemons = $db->fetchAll( + $db->select()->from('director_daemon_info')->order('fqdn')->order('username')->order('pid') + ); + + $result = []; + foreach ($daemons as $info) { + $result[] = new RunningDaemonInfo($info); + } + + return $result; + } +} diff --git a/library/Director/Daemon/DaemonDb.php b/library/Director/Daemon/DaemonDb.php new file mode 100644 index 00000000..295bf5a5 --- /dev/null +++ b/library/Director/Daemon/DaemonDb.php @@ -0,0 +1,317 @@ +details = $details; + $this->dbConfig = $dbConfig; + } + + public function register(DbBasedComponent $component) + { + $this->registeredComponents[] = $component; + + return $this; + } + + public function setConfigWatch(DbResourceConfigWatch $configWatch) + { + $this->configWatch = $configWatch; + $configWatch->notify(function ($config) { + $this->disconnect()->then(function () use ($config) { + return $this->onNewConfig($config); + }); + }); + if ($this->loop) { + $configWatch->run($this->loop); + } + + return $this; + } + + public function run(LoopInterface $loop) + { + $this->loop = $loop; + $this->connect(); + $this->refreshTimer = $loop->addPeriodicTimer(3, function () { + $this->refreshMyState(); + }); + if ($this->configWatch) { + $this->configWatch->run($this->loop); + } + } + + protected function onNewConfig($config) + { + if ($config === null) { + if ($this->dbConfig === null) { + Logger::error('DB configuration is not valid'); + } else { + Logger::error('DB configuration is no longer valid'); + } + $this->emitStatus('there is no valid DB configuration'); + $this->dbConfig = $config; + + return new FulfilledPromise(); + } else { + $this->emitStatus('configuration loaded'); + $this->dbConfig = $config; + + return $this->establishConnection($config); + } + } + + protected function establishConnection($config) + { + if ($this->connection !== null) { + Logger::error('Trying to establish a connection while being connected'); + return new RejectedPromise(); + } + $callback = function () use ($config) { + $this->reallyEstablishConnection($config); + }; + $onSuccess = function () { + $this->pendingReconnection = null; + $this->onConnected(); + }; + if ($this->pendingReconnection) { + $this->pendingReconnection->reset(); + $this->pendingReconnection = null; + } + + return $this->pendingReconnection = RetryUnless::succeeding($callback) + ->setInterval(0.2) + ->slowDownAfter(10, 10) + ->run($this->loop) + ->then($onSuccess) + ; + } + + protected function reallyEstablishConnection($config) + { + $connection = new Db(new ConfigObject($config)); + $connection->getDbAdapter()->getConnection(); + $migrations = new Migrations($connection); + if (! $migrations->hasSchema()) { + $this->emit('status', ['DB has no schema', 'error']); + throw new RuntimeException('DB has no schema'); + } + $this->wipeOrphanedInstances($connection); + if ($this->hasAnyOtherActiveInstance($connection)) { + throw new RuntimeException('DB is locked by a running daemon instance'); + } + $this->details->set('schema_version', $migrations->getLastMigrationNumber()); + + $this->connection = $connection; + $this->db = $connection->getDbAdapter(); + $this->loop->futureTick(function () { + $this->refreshMyState(); + }); + + return $connection; + } + + protected function onConnected() + { + $this->emitStatus('connected'); + Logger::info('Connected to the database'); + foreach ($this->registeredComponents as $component) { + $component->initDb($this->connection); + } + } + + /** + * @return \React\Promise\PromiseInterface + */ + protected function reconnect() + { + return $this->disconnect()->then(function () { + return $this->connect(); + }, function (Exception $e) { + Logger::error('Disconnect failed. This should never happen: ' . $e->getMessage()); + exit(1); + }); + } + + /** + * @return \React\Promise\ExtendedPromiseInterface + */ + public function connect() + { + if ($this->connection === null) { + if ($this->dbConfig) { + return $this->establishConnection($this->dbConfig); + } + } + + return new FulfilledPromise(); + } + + /** + * @return \React\Promise\ExtendedPromiseInterface + */ + public function disconnect() + { + if (! $this->connection) { + return new FulfilledPromise(); + } + if ($this->pendingDisconnect) { + return $this->pendingDisconnect->promise(); + } + + $this->eventuallySetStopped(); + $this->pendingDisconnect = new Deferred(); + $pendingComponents = new SplObjectStorage(); + foreach ($this->registeredComponents as $component) { + $pendingComponents->attach($component); + $resolve = function () use ($pendingComponents, $component) { + $pendingComponents->detach($component); + if ($pendingComponents->count() === 0) { + $this->pendingDisconnect->resolve(); + } + }; + // TODO: What should we do in case they don't? + $component->stopDb()->then($resolve); + } + + try { + if ($this->db) { + $this->db->closeConnection(); + } + + } catch (Exception $e) { + Logger::error('Failed to disconnect: ' . $e->getMessage()); + } + + return $this->pendingDisconnect->promise()->then(function () { + $this->connection = null; + $this->db = null; + $this->pendingDisconnect = null; + }); + } + + protected function emitStatus($message, $level = 'info') + { + $this->emit('status', [$message, $level]); + + return $this; + } + + protected function hasAnyOtherActiveInstance(Db $connection) + { + $db = $connection->getDbAdapter(); + + return (int) $db->fetchOne( + $db->select() + ->from('director_daemon_info', 'COUNT(*)') + ->where('ts_stopped IS NULL') + ) > 0; + } + + protected function wipeOrphanedInstances(Db $connection) + { + $db = $connection->getDbAdapter(); + $db->delete('director_daemon_info', 'ts_stopped IS NOT NULL'); + $db->delete('director_daemon_info', $db->quoteInto( + 'instance_uuid_hex = ?', + $this->details->getInstanceUuid() + )); + $count = $db->delete( + 'director_daemon_info', + 'ts_stopped IS NULL AND ts_last_update < ' . ( + DaemonUtil::timestampWithMilliseconds() - (60 * 1000) + ) + ); + if ($count > 1) { + Logger::error("Removed $count orphaned daemon instance(s) from DB"); + } + } + + protected function refreshMyState() + { + if ($this->db === null || $this->pendingReconnection || $this->pendingDisconnect) { + return; + } + try { + $updated = $this->db->update( + 'director_daemon_info', + $this->details->getPropertiesToUpdate(), + $this->db->quoteInto('instance_uuid_hex = ?', $this->details->getInstanceUuid()) + ); + + if (! $updated) { + $this->db->insert( + 'director_daemon_info', + $this->details->getPropertiesToInsert() + ); + } + } catch (Exception $e) { + Logger::error($e->getMessage()); + $this->reconnect(); + } + } + + protected function eventuallySetStopped() + { + try { + if (! $this->db) { + return; + } + $this->db->update( + 'director_daemon_info', + ['ts_stopped' => DaemonUtil::timestampWithMilliseconds()], + $this->db->quoteInto('instance_uuid_hex = ?', $this->details->getInstanceUuid()) + ); + } catch (Exception $e) { + Logger::error('Failed to update daemon info (setting ts_stopped): ' . $e->getMessage()); + } + } +} diff --git a/library/Director/Daemon/DaemonProcessDetails.php b/library/Director/Daemon/DaemonProcessDetails.php new file mode 100644 index 00000000..454e31f6 --- /dev/null +++ b/library/Director/Daemon/DaemonProcessDetails.php @@ -0,0 +1,122 @@ +instanceUuid = $instanceUuid; + $this->initialize(); + } + + public function getInstanceUuid() + { + return $this->instanceUuid; + } + + public function getPropertiesToInsert() + { + return $this->getPropertiesToUpdate() + (array) $this->info; + } + + public function getPropertiesToUpdate() + { + return [ + 'ts_last_update' => DaemonUtil::timestampWithMilliseconds(), + 'ts_stopped' => null, + 'process_info' => \json_encode($this->collectProcessInfo()), + ]; + } + + public function set($property, $value) + { + if (\property_exists($this->info, $property)) { + $this->info->$property = $value; + } else { + throw new \InvalidArgumentException("Trying to set invalid daemon info property: $property"); + } + } + + public function registerProcessList(ProcessList $list) + { + $refresh = function (Process $process) { + $this->refreshProcessInfo(); + }; + $list->on('start', $refresh)->on('exit', $refresh); + $this->processLists[] = $list; + + return $this; + } + + protected function refreshProcessInfo() + { + $this->set('process_info', \json_encode($this->collectProcessInfo())); + } + + protected function collectProcessInfo() + { + $info = (object) [$this->myPid => (object) [ + 'command' => implode(' ', $this->myArgs), + 'running' => true, + 'memory' => Memory::getUsageForPid($this->myPid) + ]]; + + foreach ($this->processLists as $processList) { + foreach ($processList->getOverview() as $pid => $details) { + $info->$pid = $details; + } + } + + return $info; + } + + protected function initialize() + { + global $argv; + CliProcess::getInitialCwd(); + $this->myArgs = $argv; + $this->myPid = \posix_getpid(); + if (isset($_SERVER['_'])) { + $self = $_SERVER['_']; + } else { + // Process does a better job, but want the relative path (if such) + $self = $_SERVER['PHP_SELF']; + } + $this->info = (object) [ + 'instance_uuid_hex' => $this->instanceUuid, + 'running_with_systemd' => 'n', + 'ts_started' => (int) ((float) $_SERVER['REQUEST_TIME_FLOAT'] * 1000), + 'ts_stopped' => null, + 'pid' => \posix_getpid(), + 'fqdn' => Platform::getFqdn(), + 'username' => Platform::getPhpUser(), + 'schema_version' => null, + 'php_version' => Platform::getPhpVersion(), + 'binary_path' => $self, + 'binary_realpath' => CliProcess::getBinaryPath(), + 'php_integer_size' => PHP_INT_SIZE, + 'php_binary_path' => PHP_BINARY, + 'php_binary_realpath' => \realpath(PHP_BINARY), // TODO: useless? + 'process_info' => null, + ]; + } +} diff --git a/library/Director/Daemon/DaemonProcessState.php b/library/Director/Daemon/DaemonProcessState.php new file mode 100644 index 00000000..8629c017 --- /dev/null +++ b/library/Director/Daemon/DaemonProcessState.php @@ -0,0 +1,85 @@ +processTitle = $processTitle; + $this->refreshMessage(); + } + + /** + * @param NotifySystemD|false $systemd + * @return $this + */ + public function setSystemd($systemd) + { + if ($systemd) { + $this->systemd = $systemd; + } else { + $this->systemd = null; + } + + return $this; + } + + public function setState($message) + { + $this->state = $message; + $this->refreshMessage(); + + return $this; + } + + public function setComponentState($name, $stateMessage) + { + if ($stateMessage === null) { + unset($this->components[$name]); + } else { + $this->components[$name] = $stateMessage; + } + $this->refreshMessage(); + } + + protected function refreshMessage() + { + $messageParts = []; + if (\strlen($this->state)) { + $messageParts[] = $this->state; + } + foreach ($this->components as $component => $message) { + $messageParts[] = "$component: $message"; + } + + $message = \implode(', ', $messageParts); + + if ($message !== $this->currentMessage) { + $this->currentMessage = $message; + if (\strlen($message) === 0) { + Process::setTitle($this->processTitle); + } else { + Process::setTitle($this->processTitle . ": $message"); + } + + if ($this->systemd) { + $this->systemd->setStatus($message); + } + } + } +} diff --git a/library/Director/Daemon/DaemonUtil.php b/library/Director/Daemon/DaemonUtil.php new file mode 100644 index 00000000..c978d11f --- /dev/null +++ b/library/Director/Daemon/DaemonUtil.php @@ -0,0 +1,16 @@ +loop = $loop; + $this->running = new ProcessList($loop); + } + + public function forwardLog(LogProxy $logProxy) + { + $this->logProxy = $logProxy; + + return $this; + } + + /** + * @param Db $db + * @return \React\Promise\ExtendedPromiseInterface + */ + public function initDb(Db $db) + { + $this->db = $db; + $check = function () { + try { + $this->checkForPendingJobs(); + $this->runNextPendingJob(); + } catch (\Exception $e) { + Logger::error($e->getMessage()); + } + }; + if ($this->timer === null) { + $this->loop->futureTick($check); + } + if ($this->timer !== null) { + Logger::info('Cancelling former timer'); + $this->loop->cancelTimer($this->timer); + } + $this->timer = $this->loop->addPeriodicTimer($this->checkInterval, $check); + + return new FulfilledPromise(); + } + + /** + * @return \React\Promise\ExtendedPromiseInterface + */ + public function stopDb() + { + $this->scheduledIds = []; + if ($this->timer !== null) { + $this->loop->cancelTimer($this->timer); + $this->timer = null; + } + $allFinished = $this->running->killOrTerminate(); + foreach ($this->runningIds as $id => $promise) { + $promise->cancel(); + } + $this->runningIds = []; + + return $allFinished; + } + + protected function hasBeenDisabled() + { + $db = $this->db->getDbAdapter(); + return $db->fetchOne( + $db->select() + ->from('director_setting', 'setting_value') + ->where('setting_name = ?', 'disable_all_jobs') + ) === 'y'; + } + + protected function checkForPendingJobs() + { + if ($this->hasBeenDisabled()) { + $this->scheduledIds = []; + // TODO: disable jobs currently going on? + return; + } + if (empty($this->scheduledIds)) { + $this->loadNextIds(); + } + } + + protected function runNextPendingJob() + { + if ($this->timer === null) { + // Reset happened. Stopping? + return; + } + + if (! empty($this->runningIds)) { + return; + } + while (! empty($this->scheduledIds)) { + if ($this->runNextJob()) { + break; + } + } + } + + protected function loadNextIds() + { + $db = $this->db->getDbAdapter(); + + foreach ($db->fetchCol( + $db->select()->from('director_job', 'id')->where('disabled = ?', 'n') + ) as $id) { + $this->scheduledIds[] = (int) $id; + }; + } + + /** + * @return bool + */ + protected function runNextJob() + { + $id = \array_shift($this->scheduledIds); + try { + $job = DirectorJob::loadWithAutoIncId((int) $id, $this->db); + if ($job->shouldRun()) { + $this->runJob($job); + return true; + } + } catch (\Exception $e) { + Logger::error('Trying to schedule Job failed: ' . $e->getMessage()); + } + + return false; + } + + /** + * @param DirectorJob $job + */ + protected function runJob(DirectorJob $job) + { + $id = $job->get('id'); + $jobName = $job->get('job_name'); + Logger::debug("Job starting: $jobName"); + $arguments = [ + 'director', + 'job', + 'run', + '--id', + $job->get('id'), + '--debug', + '--rpc' + ]; + $cli = new IcingaCliRpc(); + $cli->setArguments($arguments); + $cli->on('start', function (Process $process) { + $this->onProcessStarted($process); + }); + + // Happens on protocol (Netstring) errors or similar: + $cli->on('error', function (\Exception $e) { + Logger::error('UNEXPECTED: ' . rtrim($e->getMessage())); + }); + if ($this->logProxy) { + $logger = clone($this->logProxy); + $logger->setPrefix("[$jobName]: "); + $cli->rpc()->setHandler($this->logProxy, 'logger'); + } + unset($this->scheduledIds[$id]); + $this->runningIds[$id] = $cli->run($this->loop)->then(function () use ($id, $jobName) { + Logger::debug("Job finished: $jobName"); + })->otherwise(function (\Exception $e) use ($id, $jobName) { + Logger::error('Job failed: ' . $e->getMessage()); + })->otherwise(function (FinishedProcessState $state) { + Logger::error($state->getReason()); + })->always(function () use ($id) { + unset($this->runningIds[$id]); + $this->loop->futureTick(function () { + $this->runNextPendingJob(); + }); + }); + } + + /** + * @return ProcessList + */ + public function getProcessList() + { + return $this->running; + } + + protected function onProcessStarted(Process $process) + { + $this->running->attach($process); + } + + public function __destruct() + { + $this->stopDb(); + $this->logProxy = null; + $this->loop = null; + } +} diff --git a/library/Director/Daemon/JsonRpcLogWriter.php b/library/Director/Daemon/JsonRpcLogWriter.php new file mode 100644 index 00000000..edfa23e2 --- /dev/null +++ b/library/Director/Daemon/JsonRpcLogWriter.php @@ -0,0 +1,37 @@ + 'debug', + Logger::INFO => 'info', + Logger::WARNING => 'warning', + Logger::ERROR => 'error', + ]; + + public function __construct(Connection $connection) + { + parent::__construct(new ConfigObject([])); + $this->connection = $connection; + } + + public function log($severity, $message) + { + $message = \iconv('UTF-8', 'UTF-8//IGNORE', $message); + $this->connection->sendNotification( + Notification::create('logger.log', [ + static::$severityMap[$severity], + $message + ]) + ); + } +} diff --git a/library/Director/Daemon/LogProxy.php b/library/Director/Daemon/LogProxy.php new file mode 100644 index 00000000..9a1d9f58 --- /dev/null +++ b/library/Director/Daemon/LogProxy.php @@ -0,0 +1,76 @@ +instanceUuid = $instanceUuid; + } + + public function setPrefix($prefix) + { + $this->prefix = $prefix; + + return $this; + } + + /** + * @param Db $connection + * @return \React\Promise\ExtendedPromiseInterface + */ + public function initDb(Db $connection) + { + $this->connection = $connection; + $this->db = $connection->getDbAdapter(); + + return new FulfilledPromise(); + } + + /** + * @return \React\Promise\ExtendedPromiseInterface + */ + public function stopDb() + { + $this->connection = null; + $this->db = null; + + return new FulfilledPromise(); + } + + public function log($severity, $message) + { + Logger::$severity($this->prefix . $message); + /* + // Not yet + try { + if ($this->db) { + $this->db->insert('director_daemonlog', [ + // environment/installation/db? + 'instance_uuid' => $this->instanceUuid, + 'ts_create' => DaemonUtil::timestampWithMilliseconds(), + 'level' => $severity, + 'message' => $message, + ]); + } + } catch (Exception $e) { + Logger::error($e->getMessage()); + } + */ + } +} diff --git a/library/Director/Daemon/Logger.php b/library/Director/Daemon/Logger.php new file mode 100644 index 00000000..7b246d79 --- /dev/null +++ b/library/Director/Daemon/Logger.php @@ -0,0 +1,21 @@ +setLevel($level) + ->writer = $writer; + } catch (ConfigurationError $e) { + self::$instance->error($e->getMessage()); + } + } +} diff --git a/library/Director/Daemon/ProcessList.php b/library/Director/Daemon/ProcessList.php new file mode 100644 index 00000000..f90ae182 --- /dev/null +++ b/library/Director/Daemon/ProcessList.php @@ -0,0 +1,125 @@ +loop = $loop; + $this->processes = new \SplObjectStorage(); + foreach ($processes as $process) { + $this->attach($process); + } + } + + public function attach(Process $process) + { + $this->processes->attach($process); + $this->emit('start', [$process]); + $process->on('exit', function () use ($process) { + $this->detach($process); + $this->emit('exit', [$process]); + }); + + return $this; + } + + public function detach(Process $process) + { + $this->processes->detach($process); + + return $this; + } + + /** + * @param int $timeout + * @return \React\Promise\ExtendedPromiseInterface + */ + public function killOrTerminate($timeout = 5) + { + if ($this->processes->count() === 0) { + return new FulfilledPromise(); + } + $deferred = new Deferred(); + $killTimer = $this->loop->addTimer($timeout, function () use ($deferred) { + /** @var Process $process */ + foreach ($this->processes as $process) { + $pid = $process->getPid(); + Logger::error("Process $pid is still running, sending SIGKILL"); + $process->terminate(SIGKILL); + } + + // Let's a little bit of delay after KILLing + $this->loop->addTimer(0.1, function () use ($deferred) { + $deferred->resolve(); + }); + }); + + $timer = $this->loop->addPeriodicTimer($timeout / 20, function () use ( + $deferred, + & $timer, + $killTimer + ) { + $stopped = []; + /** @var Process $process */ + foreach ($this->processes as $process) { + if (! $process->isRunning()) { + $stopped[] = $process; + } + } + foreach ($stopped as $process) { + $this->processes->detach($process); + } + if ($this->processes->count() === 0) { + $this->loop->cancelTimer($timer); + $this->loop->cancelTimer($killTimer); + $deferred->resolve(); + } + }); + /** @var Process $process */ + foreach ($this->processes as $process) { + $process->terminate(SIGTERM); + } + + return $deferred->promise(); + } + + public function getOverview() + { + $info = []; + + /** @var Process $process */ + foreach ($this->processes as $process) { + $pid = $process->getPid(); + $info[$pid] = (object) [ + 'command' => preg_replace('/^exec /', '', $process->getCommand()), + 'running' => $process->isRunning(), + 'memory' => Memory::getUsageForPid($pid) + ]; + } + + return $info; + } +} diff --git a/library/Director/Daemon/RunningDaemonInfo.php b/library/Director/Daemon/RunningDaemonInfo.php new file mode 100644 index 00000000..adb3549d --- /dev/null +++ b/library/Director/Daemon/RunningDaemonInfo.php @@ -0,0 +1,154 @@ +setInfo($info); + } + + public function setInfo($info) + { + if (empty($info)) { + $this->info = $this->createEmptyInfo(); + } else { + $this->info = $info; + } + + return $this; + } + + public function isRunning() + { + return $this->getPid() !== null && ! $this->isOutdated(); + } + + public function getPid() + { + return (int) $this->info->pid; + } + + public function getUsername() + { + return $this->info->username; + } + + public function getFqdn() + { + return $this->info->fqdn; + } + + public function getLastUpdate() + { + return $this->info->ts_last_update; + } + + public function getLastModification() + { + return $this->info->ts_last_modification; + } + + public function getPhpVersion() + { + return $this->info->php_version; + } + + public function hasBeenStopped() + { + return $this->getTimestampStopped() !== null; + } + + public function getTimestampStarted() + { + return $this->info->ts_started; + } + + public function getTimestampStopped() + { + return $this->info->ts_stopped; + } + + public function isOutdated($seconds = 5) + { + return ( + DaemonUtil::timestampWithMilliseconds() - $this->info->ts_last_update + ) > $seconds * 1000; + } + + public function isRunningWithSystemd() + { + return $this->info->running_with_systemd === 'y'; + } + + public function getBinaryPath() + { + return $this->info->binary_path; + } + + public function getBinaryRealpath() + { + return $this->info->binary_realpath; + } + + public function binaryRealpathDiffers() + { + return $this->getBinaryPath() !== $this->getBinaryRealpath(); + } + + public function getPhpBinaryPath() + { + return $this->info->php_binary_path; + } + + public function getPhpBinaryRealpath() + { + return $this->info->php_binary_realpath; + } + + public function phpBinaryRealpathDiffers() + { + return $this->getPhpBinaryPath() !== $this->getPhpBinaryRealpath(); + } + + public function getPhpIntegerSize() + { + return (int) $this->info->php_integer_size; + } + + public function has64bitIntegers() + { + return $this->getPhpIntegerSize() === 8; + } + + /* + // TODO: not yet + public function isMaster() + { + return $this->info->is_master === 'y'; + } + + public function isStandby() + { + return ! $this->isMaster(); + } + */ + + protected function createEmptyInfo() + { + return (object) [ + 'pid' => null, + 'fqdn' => null, + 'username' => null, + 'php_version' => null, + // 'is_master' => null, + // Only if not running. Does this make any sense in 'empty info'? + 'ts_last_update' => null, + 'ts_last_modification' => null + ]; + } +} diff --git a/library/Director/Daemon/SystemdLogWriter.php b/library/Director/Daemon/SystemdLogWriter.php new file mode 100644 index 00000000..8b64442e --- /dev/null +++ b/library/Director/Daemon/SystemdLogWriter.php @@ -0,0 +1,27 @@ + 7, + Logger::INFO => 6, + Logger::WARNING => 4, + Logger::ERROR => 3, + ]; + + public function __construct() + { + parent::__construct(new ConfigObject([])); + } + + public function log($severity, $message) + { + $severity = self::$severityMap[$severity]; + echo "<$severity>$message\n"; + } +} diff --git a/library/Director/Job/JobRunner.php b/library/Director/Job/JobRunner.php deleted file mode 100644 index 4e385238..00000000 --- a/library/Director/Job/JobRunner.php +++ /dev/null @@ -1,51 +0,0 @@ -db = $db; - } - - public function runPendingJobs() - { - foreach ($this->getConfiguredJobs() as $job) { - if ($job->shouldRun()) { - Logger::info('Director JobRunner is starting "%s"', $job->job_name); - $this->run($job); - } - } - } - - protected function run(DirectorJob $job) - { - if ($this->shouldFork()) { - $this->fork($job); - } else { - $job->run(); - } - } - - protected function fork(DirectorJob $job) - { - $cmd = 'icingacli director job run ' . $job->id; - $output = `$cmd`; - // TODO: capture output - } - - protected function shouldFork() - { - return true; - } - - protected function getConfiguredJobs() - { - return DirectorJob::loadAll($this->db); - } -} diff --git a/library/Director/Objects/DirectorJob.php b/library/Director/Objects/DirectorJob.php index fbb8da34..4746d85b 100644 --- a/library/Director/Objects/DirectorJob.php +++ b/library/Director/Objects/DirectorJob.php @@ -3,6 +3,7 @@ namespace Icinga\Module\Director\Objects; use Icinga\Exception\NotFoundError; +use Icinga\Module\Director\Daemon\Logger; use Icinga\Module\Director\Data\Db\DbObjectWithSettings; use Icinga\Module\Director\Db; use Icinga\Module\Director\DirectorObject\Automation\ExportInterface; @@ -79,15 +80,20 @@ class DirectorJob extends DbObjectWithSettings implements ExportInterface try { $job->run(); $this->set('last_attempt_succeeded', 'y'); + $success = true; } catch (Exception $e) { + Logger::error($e->getMessage()); $this->set('ts_last_error', date('Y-m-d H:i:s')); $this->set('last_error_message', $e->getMessage()); $this->set('last_attempt_succeeded', 'n'); + $success = false; } if ($this->hasBeenModified()) { $this->store(); } + + return $success; } /** diff --git a/library/Director/Web/Tabs/MainTabs.php b/library/Director/Web/Tabs/MainTabs.php new file mode 100644 index 00000000..8e14e0d5 --- /dev/null +++ b/library/Director/Web/Tabs/MainTabs.php @@ -0,0 +1,85 @@ +auth = $auth; + $this->dbResourceName = $dbResourceName; + $this->add('main', [ + 'label' => $this->translate('Overview'), + 'url' => 'director' + ]); + if ($this->auth->hasPermission('director/admin')) { + $this->add('health', [ + 'label' => $this->translate('Health'), + 'url' => 'director/health' + ])->add('daemon', [ + 'label' => $this->translate('Daemon'), + 'url' => 'director/daemon' + ]); + } + } + + public function render() + { + if ($this->auth->hasPermission('director/admin')) { + if ($this->getActiveName() !== 'health') { + $state = $this->getHealthState(); + if ($state->isProblem()) { + $this->get('health')->setTagParams([ + 'class' => 'state-' . strtolower($state->getName()) + ]); + } + } + + if ($this->getActiveName() !== 'daemon') { + try { + $daemon = new BackgroundDaemonState(Db::fromResourceName($this->dbResourceName)); + if ($daemon->isRunning()) { + $state = 'ok'; + } else { + $state = 'warning'; + } + } catch (\Exception $e) { + $state = 'unknown'; + } + if ($state !== 'ok') { + $this->get('daemon')->setTagParams([ + 'class' => 'state-' . $state + ]); + } + } + } + + return parent::render(); + } + + /** + * @return \Icinga\Module\Director\CheckPlugin\PluginState + */ + protected function getHealthState() + { + $health = new Health(); + $health->setDbResourceName($this->dbResourceName); + $output = new HealthCheckPluginOutput($health); + + return $output->getState(); + } +} diff --git a/library/Director/Web/Widget/BackgroundDaemonDetails.php b/library/Director/Web/Widget/BackgroundDaemonDetails.php new file mode 100644 index 00000000..53d65452 --- /dev/null +++ b/library/Director/Web/Widget/BackgroundDaemonDetails.php @@ -0,0 +1,130 @@ +info = $info; + $this->daemon = $daemon; + } + + protected function assemble() + { + $info = $this->info; + if ($info->hasBeenStopped()) { + $this->add(Html::tag('p', [ + 'class' => 'state-hint error' + ], Html::sprintf( + $this->translate( + 'Daemon has been stopped %s, was running with PID %s as %s@%s' + ), + // $info->getHexUuid(), + $this->timeAgo($info->getTimestampStopped() / 1000), + Html::tag('strong', (string) $info->getPid()), + Html::tag('strong', $info->getUsername()), + Html::tag('strong', $info->getFqdn()) + ))); + } elseif ($info->isOutdated()) { + $this->add(Html::tag('p', [ + 'class' => 'state-hint error' + ], Html::sprintf( + $this->translate( + 'Daemon keep-alive is outdated, was last seen running with PID %s as %s@%s %s' + ), + // $info->getHexUuid(), + Html::tag('strong', (string) $info->getPid()), + Html::tag('strong', $info->getUsername()), + Html::tag('strong', $info->getFqdn()), + $this->timeAgo($info->getLastUpdate() / 1000) + ))); + } else { + $details = new NameValueTable(); + $details->addNameValuePairs([ + $this->translate('Startup Time') => DateFormatter::formatDateTime($info->getTimestampStarted() / 1000), + $this->translate('PID') => $info->getPid(), + $this->translate('Username') => $info->getUsername(), + $this->translate('FQDN') => $info->getFqdn(), + $this->translate('Running with systemd') => $info->isRunningWithSystemd() + ? $this->translate('yes') + : $this->translate('no'), + $this->translate('Binary') => $info->getBinaryPath() + . ($info->binaryRealpathDiffers() ? ' -> ' . $info->getBinaryRealpath() : ''), + $this->translate('PHP Binary') => $info->getPhpBinaryPath() + . ($info->phpBinaryRealpathDiffers() ? ' -> ' . $info->getPhpBinaryRealpath() : ''), + $this->translate('PHP Version') => $info->getPhpVersion(), + $this->translate('PHP Integer') => $info->has64bitIntegers() + ? '64bit' + : Html::sprintf( + '%sbit (%s)', + $info->getPhpIntegerSize() * 8, + Html::tag('span', ['class' => 'error'], $this->translate('unsupported')) + ), + ]); + $this->add($details); + $this->add(Html::tag('p', [ + 'class' => 'state-hint ok' + ], Html::sprintf( + $this->translate( + 'Daemon is running with PID %s as %s@%s, last refresh happened %s' + ), + // $info->getHexUuid(), + Html::tag('strong', (string)$info->getPid()), + Html::tag('strong', $info->getUsername()), + Html::tag('strong', $info->getFqdn()), + $this->timeAgo($info->getLastUpdate() / 1000) + ))); + + $this->add(Html::tag('h2', $this->translate('Process List'))); + $processes = \json_decode($this->daemon->process_info); + $table = new Table(); + $table->add(Html::tag('thead', Html::tag('tr', Html::wrapEach([ + 'PID', + 'Command', + 'Memory' + ], 'th')))); + $table->setAttribute('class', 'common-table'); + foreach ($processes as $pid => $process) { + $table->add($table::row([ + [ + Icon::create($process->running ? 'ok' : 'warning-empty'), + ' ', + $pid + ], + Html::tag('pre', $process->command), + Format::bytes($process->memory->rss) + ])); + } + $this->add($table); + } + } + + protected function timeAgo($time) + { + return Html::tag('span', [ + 'class' => 'time-ago', + 'title' => DateFormatter::formatDateTime($time) + ], DateFormatter::timeAgo($time)); + } +} diff --git a/library/Director/Web/Widget/Documentation.php b/library/Director/Web/Widget/Documentation.php new file mode 100644 index 00000000..4420f849 --- /dev/null +++ b/library/Director/Web/Widget/Documentation.php @@ -0,0 +1,86 @@ +app = $app; + $this->auth = $auth; + } + + public function getModuleLink($label, $module, $chapter, $title = null) + { + if ($title !== null) { + $title = sprintf( + $this->translate('Click to read our documentation: %s'), + $title + ); + } + $linkToGitHub = false; + $hasModule = $this->app->getModuleManager()->hasLoaded($module); + if ($hasModule && $this->hasAccessToDocumentationModule()) { + return Link::create( + $label, + 'doc/module/director/chapter/' . \preg_replace('/^\d+-/', '', \rawurlencode($chapter)), + null, + [ + 'data-base-target' => '_next', + 'class' => 'icon-book', + 'title' => $title, + ] + ); + } elseif ($linkToGitHub) { + return Html::tag('a', [ + 'href' => $this->githubDocumentationUrl($module, $chapter), + 'target' => '_blank', + 'title' => $title, + ], $label); + } else { + return Html::tag('a', [ + 'href' => $this->icingaDocumentationUrl($module, $chapter), + 'target' => '_blank', + 'title' => $title, + ], $label); + } + } + + protected function githubDocumentationUrl($module, $chapter) + { + return sprintf( + "https://github.com/Icinga/icingaweb2-module-%s/blob/master/doc/%s.md", + \rawurlencode($module), + \rawurlencode($chapter) + ); + } + + protected function icingaDocumentationUrl($module, $chapter) + { + return sprintf( + 'https://icinga.com/docs/%s/latest/doc/%s/', + \rawurlencode($module), + \rawurlencode($chapter) + ); + } + + protected function hasAccessToDocumentationModule() + { + return $this->app->getModuleManager()->hasLoaded('doc') + && $this->auth->hasPermission('module/doc'); + } +} diff --git a/library/Director/Web/Widget/ImportSourceDetails.php b/library/Director/Web/Widget/ImportSourceDetails.php index 57e3e569..736663a9 100644 --- a/library/Director/Web/Widget/ImportSourceDetails.php +++ b/library/Director/Web/Widget/ImportSourceDetails.php @@ -20,9 +20,6 @@ class ImportSourceDetails extends HtmlDocument $this->source = $source; } - /** - * @throws \Icinga\Exception\IcingaException - */ protected function assemble() { $source = $this->source; @@ -35,7 +32,7 @@ class ImportSourceDetails extends HtmlDocument case 'unknown': $this->add(Html::tag( 'p', - null, + ['class' => 'state-hint warning'], $this->translate( "It's currently unknown whether we are in sync with this Import Source." . ' You should either check for changes or trigger a new Import Run.' @@ -43,7 +40,7 @@ class ImportSourceDetails extends HtmlDocument )); break; case 'in-sync': - $this->add(Html::tag('p', null, sprintf( + $this->add(Html::tag('p', ['class' => 'state-hint ok'], sprintf( $this->translate( 'This Import Source was last found to be in sync at %s.' ), @@ -54,13 +51,13 @@ class ImportSourceDetails extends HtmlDocument // - there have been activities since then break; case 'pending-changes': - $this->add(Html::tag('p', ['class' => 'warning'], $this->translate( + $this->add(Html::tag('p', ['class' => 'state-hint warning'], $this->translate( 'There are pending changes for this Import Source. You should trigger a new' . ' Import Run.' ))); break; case 'failing': - $this->add(Html::tag('p', ['class' => 'error'], sprintf( + $this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf( $this->translate( 'This Import Source failed when last checked at %s: %s' ), @@ -69,7 +66,7 @@ class ImportSourceDetails extends HtmlDocument ))); break; default: - $this->add(Html::tag('p', ['class' => 'error'], sprintf( + $this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf( $this->translate('This Import Source has an invalid state: %s'), $source->get('import_state') ))); diff --git a/library/Director/Web/Widget/JobDetails.php b/library/Director/Web/Widget/JobDetails.php index 32e8ebd9..3be00ca1 100644 --- a/library/Director/Web/Widget/JobDetails.php +++ b/library/Director/Web/Widget/JobDetails.php @@ -2,6 +2,7 @@ namespace Icinga\Module\Director\Web\Widget; +use Icinga\Date\DateFormatter; use ipl\Html\HtmlDocument; use Icinga\Module\Director\Objects\DirectorJob; use ipl\Html\Html; @@ -18,43 +19,52 @@ class JobDetails extends HtmlDocument */ public function __construct(DirectorJob $job) { - if ($job->disabled === 'y') { - $this->add(Html::tag('p', ['class' => 'error'], sprintf( + $runInterval = $job->get('run_interval'); + if ($job->hasBeenDisabled()) { + $this->add(Html::tag('p', ['class' => 'state-hint error'], sprintf( $this->translate( 'This job would run every %ds. It has been disabled and will' . ' therefore not be executed as scheduled' ), - $job->run_interval + $runInterval ))); } else { //$class = $job->job(); echo $class::getDescription() $msg = $job->isPending() ? sprintf( $this->translate('This job runs every %ds and is currently pending'), - $job->run_interval + $runInterval ) : sprintf( $this->translate('This job runs every %ds.'), - $job->run_interval + $runInterval ); $this->add(Html::tag('p', null, $msg)); } - if ($job->ts_last_attempt) { - if ($job->last_attempt_succeeded) { - $this->add(Html::tag('p', null, sprintf( - $this->translate('The last attempt succeeded at %s'), - $job->ts_last_attempt + $tsLastAttempt = $job->get('ts_last_attempt'); + $ts = \strtotime($tsLastAttempt); + $timeAgo = Html::tag('span', [ + 'class' => 'time-ago', + 'title' => DateFormatter::formatDateTime($ts) + ], DateFormatter::timeAgo($ts)); + if ($tsLastAttempt) { + if ($job->get('last_attempt_succeeded') === 'y') { + $this->add(Html::tag('p', ['class' => 'state-hint ok'], Html::sprintf( + $this->translate('The last attempt succeeded %s'), + $timeAgo ))); } else { - $this->add(Html::tag('p', ['class' => 'error'], sprintf( - $this->translate('The last attempt failed at %s: %s'), - $job->ts_last_attempt, - $job->ts_last_error + $this->add(Html::tag('p', ['class' => 'state-hint error'], Html::sprintf( + $this->translate('The last attempt failed %s: %s'), + $timeAgo, + $job->get('last_error_message') ))); } } else { - $this->add(Html::tag('p', null, $this->translate('This job has not been executed yet'))); + $this->add(Html::tag('p', [ + 'class' => 'state-hint warning' + ], $this->translate('This job has not been executed yet'))); } } } diff --git a/module.info b/module.info index 51bbbeae..bcb267e1 100644 --- a/module.info +++ b/module.info @@ -1,6 +1,6 @@ Name: Icinga Director Version: master -Depends: reactbundle (>=0.6.0), ipl (>=0.3.0), incubator (>=0.3.0) +Depends: reactbundle (>=0.7.0), ipl (>=0.3.0), incubator (>=0.4.0) Description: Director - Config tool for Icinga 2 Icinga Director is a configuration tool that has been designed to make Icinga 2 configuration easy and understandable. diff --git a/public/css/module.less b/public/css/module.less index 4b686acd..e74e0f47 100644 --- a/public/css/module.less +++ b/public/css/module.less @@ -7,6 +7,10 @@ div.action-bar a:focus, .tabs a:focus { } } +a:before { + text-decoration: none; +} + form:focus { outline: none; } @@ -1075,7 +1079,7 @@ span.error { } } -p.error { +p.error:not(.state-hint) { color: white; padding: 1em 2em; background-color: @colorCritical; @@ -1087,7 +1091,7 @@ p.error { } } -p.warning { +p.warning:not(.state-hint) { color: white; padding: 1em 2em; background-color: @colorWarning; @@ -1099,7 +1103,7 @@ p.warning { } } -p.information { +p.information:not(.state-hint) { color: white; padding: 1em 2em; background-color: @colorOk; @@ -1111,6 +1115,45 @@ p.information { } } +p.state-hint { + border: 1px solid @text-color; + padding: 0.5em; + line-height: 2em; + max-width: 60em; + border-left-width: 3em; + &:before { + position: relative; + margin-left: -1.5em; + margin-right: 0.5em; + height: 100%; + vertical-align: middle; + font-family: 'ifont'; + color: white; + font-size: 2em; + } + &.ok { + border-color: @color-ok; + } + &.warning { + border-color: @color-warning; + } + &.error { + border-color: @color-critical; + } + &.critical:before, &.error:before { + content: '\e885'; + } + &.warning:before { + content: '\e885'; + } + &.ok:before { + content: '\e803'; + } + a { + text-decoration: underline; + } +} + table th.actions, table td.actions { text-align: right; }