From 7f35845944820fdf7c33ea4d6d5ff72b8f5af35b Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 2 Nov 2021 12:47:36 +0100 Subject: [PATCH 1/4] Notifications if syncqueue exceeds limits in MC --- .../include/class/ConsoleSupervisor.php | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/pandora_console/include/class/ConsoleSupervisor.php b/pandora_console/include/class/ConsoleSupervisor.php index f6e6da12d7..c91fa807e4 100644 --- a/pandora_console/include/class/ConsoleSupervisor.php +++ b/pandora_console/include/class/ConsoleSupervisor.php @@ -52,6 +52,12 @@ class ConsoleSupervisor */ public const MIN_PERFORMANCE_MODULES = 100; + + /** + * Minimum queued elements in synchronization queue to be warned.. + */ + public const MIN_SYNC_QUEUE_LENGTH = 200; + /** * Show if console supervisor is enabled or not. * @@ -242,6 +248,15 @@ class ConsoleSupervisor $this->checkAuditLogOldLocation(); + /* + * Checks if sync queue is longer than limits. + * NOTIF.SYNCQUEUE.LENGTH + */ + + if (is_metaconsole() === true) { + $this->checkSyncQueue(); + } + } @@ -492,6 +507,15 @@ class ConsoleSupervisor */ $this->checkAuditLogOldLocation(); + + /* + * Checks if sync queue is longer than limits. + * NOTIF.SYNCQUEUE.LENGTH + */ + + if (is_metaconsole() === true) { + $this->checkSyncQueue(); + } } @@ -2684,4 +2708,56 @@ class ConsoleSupervisor } + /** + * Verifies the status of synchronization queue and warns if something is + * not working as expected. + * + * @return void + */ + public function checkSyncQueue() + { + if (is_metaconsole() !== true) { + return; + } + + $sync = new PandoraFMS\Enterprise\Metaconsole\Synchronizer(); + $counts = $sync->counts(); + + if (count($counts) === 0) { + // Clean all. + $this->cleanNotifications('NOTIF.SYNCQUEUE.LENGTH.%'); + } + + foreach ($counts as $node_id => $count) { + if ($count < self::MIN_SYNC_QUEUE_LENGTH) { + $this->cleanNotifications('NOTIF.SYNCQUEUE.LENGTH.'.$node_id); + } else { + try { + $node = new PandoraFMS\Enterprise\Metaconsole\Node($node_id); + + $url = '__url__/index.php?sec=advanced&sec2=advanced/metasetup&tab=consoles'; + + $this->notify( + [ + 'type' => 'NOTIF.SYNCQUEUE.LENGTH.'.$node_id, + 'title' => __('Node %s sync queue length exceeded, ', $node->server_name()), + 'message' => __( + 'Synchronization queue lenght for node %s is %d items, this value should be 0 or lower than %d, pleae visit the ', + $node->server_name(), + $count, + self::MIN_SYNC_QUEUE_LENGTH + ), + 'url' => $url, + ] + ); + } catch (\Exception $e) { + // Clean, exception in node finding. + $this->cleanNotifications('NOTIF.SYNCQUEUE.LENGTH.'.$node_id); + } + } + } + + } + + } From 976e881cb91037f1b948762275d9968ea421b202 Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 2 Nov 2021 12:48:48 +0100 Subject: [PATCH 2/4] Message update notif.syncqueue.length --- pandora_console/include/class/ConsoleSupervisor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora_console/include/class/ConsoleSupervisor.php b/pandora_console/include/class/ConsoleSupervisor.php index c91fa807e4..20d928bac9 100644 --- a/pandora_console/include/class/ConsoleSupervisor.php +++ b/pandora_console/include/class/ConsoleSupervisor.php @@ -2742,7 +2742,7 @@ class ConsoleSupervisor 'type' => 'NOTIF.SYNCQUEUE.LENGTH.'.$node_id, 'title' => __('Node %s sync queue length exceeded, ', $node->server_name()), 'message' => __( - 'Synchronization queue lenght for node %s is %d items, this value should be 0 or lower than %d, pleae visit the ', + 'Synchronization queue lenght for node %s is %d items, this value should be 0 or lower than %d, please check the queue status.', $node->server_name(), $count, self::MIN_SYNC_QUEUE_LENGTH From 4f2bca1df416bf5cd354ef0e1850db977c93e8d8 Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 2 Nov 2021 13:51:30 +0100 Subject: [PATCH 3/4] Added sync status notifications --- .../include/class/ConsoleSupervisor.php | 64 +++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/pandora_console/include/class/ConsoleSupervisor.php b/pandora_console/include/class/ConsoleSupervisor.php index 20d928bac9..96e46aa8cb 100644 --- a/pandora_console/include/class/ConsoleSupervisor.php +++ b/pandora_console/include/class/ConsoleSupervisor.php @@ -254,7 +254,8 @@ class ConsoleSupervisor */ if (is_metaconsole() === true) { - $this->checkSyncQueue(); + $this->checkSyncQueueLength(); + $this->checkSyncQueueStatus(); } } @@ -514,7 +515,8 @@ class ConsoleSupervisor */ if (is_metaconsole() === true) { - $this->checkSyncQueue(); + $this->checkSyncQueueLength(); + $this->checkSyncQueueStatus(); } } @@ -2714,14 +2716,14 @@ class ConsoleSupervisor * * @return void */ - public function checkSyncQueue() + public function checkSyncQueueLength() { if (is_metaconsole() !== true) { return; } $sync = new PandoraFMS\Enterprise\Metaconsole\Synchronizer(); - $counts = $sync->counts(); + $counts = $sync->getQueues(true); if (count($counts) === 0) { // Clean all. @@ -2760,4 +2762,58 @@ class ConsoleSupervisor } + /** + * Verifies the status of synchronization queue and warns if something is + * not working as expected. + * + * @return void + */ + public function checkSyncQueueStatus() + { + if (is_metaconsole() !== true) { + return; + } + + $sync = new PandoraFMS\Enterprise\Metaconsole\Synchronizer(); + $queues = $sync->getQueues(); + if (count($queues) === 0) { + // Clean all. + $this->cleanNotifications('NOTIF.SYNCQUEUE.STATUS.%'); + } + + foreach ($queues as $node_id => $queue) { + if (count($queue) === 0) { + $this->cleanNotifications('NOTIF.SYNCQUEUE.STATUS.'.$node_id); + continue; + } + + $item = $queue[0]; + + if (empty($item->error()) === false) { + try { + $node = new PandoraFMS\Enterprise\Metaconsole\Node($node_id); + $url = '__url__/index.php?sec=advanced&sec2=advanced/metasetup&tab=consoles'; + + $this->notify( + [ + 'type' => 'NOTIF.SYNCQUEUE.STATUS.'.$node_id, + 'title' => __('Node %s sync queue failed, ', $node->server_name()), + 'message' => __( + 'Node %s cannot process synchronization queue due %s, please check the queue status.', + $node->server_name(), + $item->error() + ), + 'url' => $url, + ] + ); + } catch (\Exception $e) { + // Clean, exception in node finding. + $this->cleanNotifications('NOTIF.SYNCQUEUE.STATUS.'.$node_id); + } + } + } + + } + + } From 3a74a6ea4265592a7bff76fbb344b3ca426930f4 Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 2 Nov 2021 13:59:17 +0100 Subject: [PATCH 4/4] custom val --- pandora_console/include/class/ConsoleSupervisor.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pandora_console/include/class/ConsoleSupervisor.php b/pandora_console/include/class/ConsoleSupervisor.php index 96e46aa8cb..ea0ac8479d 100644 --- a/pandora_console/include/class/ConsoleSupervisor.php +++ b/pandora_console/include/class/ConsoleSupervisor.php @@ -2718,6 +2718,8 @@ class ConsoleSupervisor */ public function checkSyncQueueLength() { + global $config; + if (is_metaconsole() !== true) { return; } @@ -2730,8 +2732,13 @@ class ConsoleSupervisor $this->cleanNotifications('NOTIF.SYNCQUEUE.LENGTH.%'); } + $items_min = $config['sync_queue_items_max']; + if (is_numeric($items_min) !== true && $items_min <= 0) { + $items_min = self::MIN_SYNC_QUEUE_LENGTH; + } + foreach ($counts as $node_id => $count) { - if ($count < self::MIN_SYNC_QUEUE_LENGTH) { + if ($count < $items_min) { $this->cleanNotifications('NOTIF.SYNCQUEUE.LENGTH.'.$node_id); } else { try { @@ -2747,7 +2754,7 @@ class ConsoleSupervisor 'Synchronization queue lenght for node %s is %d items, this value should be 0 or lower than %d, please check the queue status.', $node->server_name(), $count, - self::MIN_SYNC_QUEUE_LENGTH + $items_min ), 'url' => $url, ]