Merge pull request #3965 from Icinga/fix/slow-queries-when-filtering-groups-3928

Enhance query performance when filtering for groups and reduce double query execution

library/Icinga/Data/SimpleQuery.php

@@ -36,6 +36,13 @@ class SimpleQuery implements QueryInterface, Queryable, Iterator
      */
     protected $iteratorPosition;
 
+    /**
+     * The amount of rows previously calculated
+     *
+     * @var int
+     */
+    protected $cachedCount;
+
     /**
      * The target you are going to query
      *
@@ -450,7 +457,7 @@ class SimpleQuery implements QueryInterface, Queryable, Iterator
      */
     public function hasResult()
     {
-        return $this->iteratorPosition !== null || $this->fetchRow() !== false;
+        return $this->cachedCount > 0 || $this->iteratorPosition !== null || $this->fetchRow() !== false;
     }
 
     /**
@@ -647,6 +654,7 @@ class SimpleQuery implements QueryInterface, Queryable, Iterator
         $query->limit(0, 0);
         Benchmark::measure('Counting all results started');
         $count = $this->ds->count($query);
+        $this->cachedCount = $count;
         Benchmark::measure('Counting all results finished');
         return $count;
     }

modules/monitoring/application/views/scripts/partials/event-history.phtml

@@ -29,19 +29,17 @@ if (! $url->hasParam('page') || ($page = (int) $url->getParam('page')) < 1) {
 $history->limit($limit * $page);
 ?>
 <div class="content">
-<?php if (! $history->hasResult()): ?>
-    <p><?= $this->translate('No historical events found matching the filter.') ?></p>
-</div>
-<?php return; endif ?>
 <?php
 $dateFormat = $this->translate('%A, %B %e, %Y', 'date.verbose');
 $lastDate = null;
 $flappingMsg = $this->translate('Flapping with a %.2f%% state change rate');
 $rowAction = Url::fromPath('monitoring/event/show');
 ?>
+    <?php foreach ($history->peekAhead() as $event): ?>
+<?php if ($lastDate === null): ?>
     <table class="table-row-selectable state-table" data-base-target="_next">
-    <tbody>
+        <tbody>
-    <?php foreach ($history->peekAhead() as $event):
+<?php endif;
         $icon = '';
         $iconTitle = null;
         $isService = isset($event->service_description);
@@ -240,8 +238,10 @@ $rowAction = Url::fromPath('monitoring/event/show');
             </td>
         </tr>
         <?php endforeach ?>
+<?php if ($lastDate !== null): ?>
         </tbody>
     </table>
+<?php endif ?>
 <?php if ($history->hasMore()): ?>
     <div class="action-links">
     <?php if ($this->compact) {

modules/monitoring/library/Monitoring/Backend/Ido/Query/EmptyhostgroupQuery.php

@@ -0,0 +1,36 @@
+<?php
+/* Icinga Web 2 | (c) 2019 Icinga GmbH | GPLv2+ */
+
+namespace Icinga\Module\Monitoring\Backend\Ido\Query;
+
+class EmptyhostgroupQuery extends HostgroupQuery
+{
+    protected $subQueryTargets = [];
+
+    protected $columnMap = [
+        'hostgroups' => [
+            'hostgroup'             => 'hgo.name1 COLLATE latin1_general_ci',
+            'hostgroup_alias'       => 'hg.alias COLLATE latin1_general_ci',
+            'hostgroup_name'        => 'hgo.name1',
+            'host_name'             => '(NULL)',
+            'service_description'   => '(NULL)',
+            'servicegroup_name'     => '(NULL)'
+        ],
+        'instances' => [
+            'instance_name'         => 'i.instance_name'
+        ]
+    ];
+
+    protected function joinBaseTables()
+    {
+        parent::joinBaseTables();
+
+        $this->select->joinLeft(
+            ['ehgm' => $this->prefix . 'hostgroup_members'],
+            'ehgm.hostgroup_id = hg.hostgroup_id',
+            []
+        );
+        $this->select->group(['hgo.object_id', 'hg.hostgroup_id']);
+        $this->select->having('COUNT(ehgm.hostgroup_member_id) = ?', 0);
+    }
+}

modules/monitoring/library/Monitoring/Backend/Ido/Query/EmptyservicegroupQuery.php

@@ -0,0 +1,36 @@
+<?php
+/* Icinga Web 2 | (c) 2019 Icinga GmbH | GPLv2+ */
+
+namespace Icinga\Module\Monitoring\Backend\Ido\Query;
+
+class EmptyservicegroupQuery extends ServicegroupQuery
+{
+    protected $subQueryTargets = [];
+
+    protected $columnMap = [
+        'servicegroups' => [
+            'servicegroup'          => 'sgo.name1 COLLATE latin1_general_ci',
+            'servicegroup_alias'    => 'sg.alias COLLATE latin1_general_ci',
+            'servicegroup_name'     => 'sgo.name1',
+            'host_name'             => '(NULL)',
+            'hostgroup_name'        => '(NULL)',
+            'service_description'   => '(NULL)'
+        ],
+        'instances' => [
+            'instance_name'         => 'i.instance_name'
+        ]
+    ];
+
+    protected function joinBaseTables()
+    {
+        parent::joinBaseTables();
+
+        $this->select->joinLeft(
+            ['esgm' => $this->prefix . 'servicegroup_members'],
+            'esgm.servicegroup_id = sg.servicegroup_id',
+            []
+        );
+        $this->select->group(['sgo.object_id', 'sg.servicegroup_id']);
+        $this->select->having('COUNT(esgm.servicegroup_member_id) = ?', 0);
+    }
+}

modules/monitoring/library/Monitoring/Backend/Ido/Query/HostgroupQuery.php

@@ -59,7 +59,7 @@ class HostgroupQuery extends IdoQuery
                         END
                     END
                 END',
-            'host_state'    => 'CASE WHEN hs.has_been_checked = 0 OR (hs.has_been_checked IS NULL AND hs.hoststatus_id IS NOT NULL) THEN 99 ELSE hs.current_state END'
+            'host_state'    => 'CASE WHEN hs.has_been_checked = 0 OR hs.has_been_checked IS NULL THEN 99 ELSE hs.current_state END'
         ),
         'instances' => array(
             'instance_name' => 'i.instance_name'
@@ -112,7 +112,7 @@ class HostgroupQuery extends IdoQuery
                              END
                          END
                 END',
-            'service_state'     => 'CASE WHEN ss.has_been_checked = 0 OR (ss.has_been_checked IS NULL AND ss.servicestatus_id IS NOT NULL) THEN 99 ELSE ss.current_state END'
+            'service_state'     => 'CASE WHEN ss.has_been_checked = 0 OR ss.has_been_checked IS NULL THEN 99 ELSE ss.current_state END'
         )
     );
 
@@ -171,7 +171,7 @@ class HostgroupQuery extends IdoQuery
     protected function joinHosts()
     {
         $this->requireVirtualTable('members');
-        $this->select->joinLeft(
+        $this->select->join(
             array('h' => $this->prefix . 'hosts'),
             'h.host_object_id = ho.object_id',
             array()
@@ -184,7 +184,7 @@ class HostgroupQuery extends IdoQuery
     protected function joinHoststatus()
     {
         $this->requireVirtualTable('members');
-        $this->select->joinLeft(
+        $this->select->join(
             array('hs' => $this->prefix . 'hoststatus'),
             'hs.host_object_id = ho.object_id',
             array()
@@ -196,7 +196,7 @@ class HostgroupQuery extends IdoQuery
      */
     protected function joinInstances()
     {
-        $this->select->joinLeft(
+        $this->select->join(
             array('i' => $this->prefix . 'instances'),
             'i.instance_id = hg.instance_id',
             array()
@@ -208,11 +208,11 @@ class HostgroupQuery extends IdoQuery
      */
     protected function joinMembers()
     {
-        $this->select->joinLeft(
+        $this->select->join(
             array('hgm' => $this->prefix . 'hostgroup_members'),
             'hgm.hostgroup_id = hg.hostgroup_id',
             array()
-        )->joinLeft(
+        )->join(
             array('ho' => $this->prefix . 'objects'),
             'hgm.host_object_id = ho.object_id AND ho.is_active = 1 AND ho.objecttype_id = 1',
             array()
@@ -225,15 +225,15 @@ class HostgroupQuery extends IdoQuery
     protected function joinServicegroups()
     {
         $this->requireVirtualTable('services');
-        $this->select->joinLeft(
+        $this->select->join(
             array('sgm' => $this->prefix . 'servicegroup_members'),
             'sgm.service_object_id = s.service_object_id',
             array()
-        )->joinLeft(
+        )->join(
             array('sg' => $this->prefix . 'servicegroups'),
             'sgm.servicegroup_id = sg.servicegroup_id',
             array()
-        )->joinLeft(
+        )->join(
             array('sgo' => $this->prefix . 'objects'),
             'sgo.object_id = sg.servicegroup_object_id AND sgo.is_active = 1 AND sgo.objecttype_id = 4',
             array()
@@ -246,11 +246,11 @@ class HostgroupQuery extends IdoQuery
     protected function joinServices()
     {
         $this->requireVirtualTable('hosts');
-        $this->select->joinLeft(
+        $this->select->join(
             array('s' => $this->prefix . 'services'),
             's.host_object_id = h.host_object_id',
             array()
-        )->joinLeft(
+        )->join(
             array('so' => $this->prefix . 'objects'),
             'so.object_id = s.service_object_id AND so.is_active = 1 AND so.objecttype_id = 2',
             array()
@@ -264,7 +264,7 @@ class HostgroupQuery extends IdoQuery
     {
         $this->requireVirtualTable('services');
         $this->requireVirtualTable('hoststatus');
-        $this->select->joinLeft(
+        $this->select->join(
             array('ss' => $this->prefix . 'servicestatus'),
             'ss.service_object_id = so.object_id',
             array()

modules/monitoring/library/Monitoring/Backend/Ido/Query/HostgroupsummaryQuery.php

@@ -108,7 +108,24 @@ class HostgroupsummaryQuery extends IdoQuery
             )
         );
         $this->subQueries[] = $services;
-        $this->summaryQuery = $this->db->select()->union(array($hosts, $services), Zend_Db_Select::SQL_UNION_ALL);
+        $emptyGroups = $this->createSubQuery(
+            'Emptyhostgroup',
+            [
+                'hostgroup_alias',
+                'hostgroup_name',
+                'host_handled'      => new Zend_Db_Expr('NULL'),
+                'host_severity'     => new Zend_Db_Expr('0'),
+                'host_state'        => new Zend_Db_Expr('NULL'),
+                'service_handled'   => new Zend_Db_Expr('NULL'),
+                'service_severity'  => new Zend_Db_Expr('0'),
+                'service_state'     => new Zend_Db_Expr('NULL'),
+            ]
+        );
+        $this->subQueries[] = $emptyGroups;
+        $this->summaryQuery = $this->db->select()->union(
+            [$hosts, $services, $emptyGroups],
+            Zend_Db_Select::SQL_UNION_ALL
+        );
         $this->select->from(array('hostgroupsummary' => $this->summaryQuery), array());
         $this->group(array('hostgroup_name', 'hostgroup_alias'));
         $this->joinedVirtualTables['hostgroupsummary'] = true;

modules/monitoring/library/Monitoring/Backend/Ido/Query/IdoQuery.php

@@ -633,12 +633,6 @@ abstract class IdoQuery extends DbQuery
             new Zend_Db_Expr($ours)
         ));
 
-        if (! $negate) {
-            $subQueryFilter = $subQueryFilter->orFilter(
-                new FilterExpression($ours, '', new Zend_Db_Expr('IS NULL'))
-            );
-        }
-
         $subQuery
             ->setFilter($subQueryFilter)
             ->clearGroupingRules()

modules/monitoring/library/Monitoring/Backend/Ido/Query/ServicegroupQuery.php

@@ -89,7 +89,7 @@ class ServicegroupQuery extends IdoQuery
                              END
                          END
                 END',
-            'service_state'     => 'CASE WHEN ss.has_been_checked = 0 OR (ss.has_been_checked IS NULL AND ss.servicestatus_id IS NOT NULL) THEN 99 ELSE ss.current_state END'
+            'service_state'     => 'CASE WHEN ss.has_been_checked = 0 OR ss.has_been_checked IS NULL THEN 99 ELSE ss.current_state END'
         )
     );
 
@@ -192,15 +192,15 @@ class ServicegroupQuery extends IdoQuery
     protected function joinHostgroups()
     {
         $this->requireVirtualTable('services');
-        $this->select->joinLeft(
+        $this->select->join(
             array('hgm' => $this->prefix . 'hostgroup_members'),
             'hgm.host_object_id = s.host_object_id',
             array()
-        )->joinLeft(
+        )->join(
             array('hg' => $this->prefix . 'hostgroups'),
             'hg.hostgroup_id = hgm.hostgroup_id',
             array()
-        )->joinLeft(
+        )->join(
             array('hgo' => $this->prefix . 'objects'),
             'hgo.object_id = hg.hostgroup_object_id AND hgo.objecttype_id = 3 AND hgo.is_active = 1',
             array()
@@ -225,7 +225,7 @@ class ServicegroupQuery extends IdoQuery
      */
     protected function joinInstances()
     {
-        $this->select->joinLeft(
+        $this->select->join(
             array('i' => $this->prefix . 'instances'),
             'i.instance_id = sg.instance_id',
             array()
@@ -237,11 +237,11 @@ class ServicegroupQuery extends IdoQuery
      */
     protected function joinMembers()
     {
-        $this->select->joinLeft(
+        $this->select->join(
             array('sgm' => $this->prefix . 'servicegroup_members'),
             'sgm.servicegroup_id = sg.servicegroup_id',
             array()
-        )->joinLeft(
+        )->join(
             array('so' => $this->prefix . 'objects'),
             'so.object_id = sgm.service_object_id AND so.objecttype_id = 2 AND so.is_active = 1',
             array()
@@ -254,7 +254,7 @@ class ServicegroupQuery extends IdoQuery
     protected function joinServices()
     {
         $this->requireVirtualTable('members');
-        $this->select->joinLeft(
+        $this->select->join(
             array('s' => $this->prefix . 'services'),
             's.service_object_id = so.object_id',
             array()
@@ -267,12 +267,12 @@ class ServicegroupQuery extends IdoQuery
     protected function joinServicestatus()
     {
         $this->requireVirtualTable('services');
-        $this->select->joinLeft(
+        $this->select->join(
             array('hs' => $this->prefix . 'hoststatus'),
             'hs.host_object_id = s.host_object_id',
             array()
         );
-        $this->select->joinLeft(
+        $this->select->join(
             array('ss' => $this->prefix . 'servicestatus'),
             'ss.service_object_id = so.object_id',
             array()

modules/monitoring/library/Monitoring/Backend/Ido/Query/ServicegroupsummaryQuery.php

@@ -4,6 +4,8 @@
 namespace Icinga\Module\Monitoring\Backend\Ido\Query;
 
 use Icinga\Data\Filter\Filter;
+use Zend_Db_Expr;
+use Zend_Db_Select;
 
 /**
  * Query for service group summary
@@ -23,7 +25,7 @@ class ServicegroupsummaryQuery extends IdoQuery
             'services_ok'                                   => 'SUM(CASE WHEN service_state = 0 THEN 1 ELSE 0 END)',
             'services_pending'                              => 'SUM(CASE WHEN service_state = 99 THEN 1 ELSE 0 END)',
             'services_severity'                             => 'MAX(service_severity)',
-            'services_total'                                => 'SUM(CASE WHEN service_state IS NOT NULL THEN 1 ELSE 0 END)',
+            'services_total'                                => 'SUM(1)',
             'services_unknown'                              => 'SUM(CASE WHEN service_state = 3 THEN 1 ELSE 0 END)',
             'services_unknown_handled'                      => 'SUM(CASE WHEN service_state = 3 AND service_handled = 1 THEN 1 ELSE 0 END)',
             'services_unknown_unhandled'                    => 'SUM(CASE WHEN service_state = 3 AND service_handled = 0 THEN 1 ELSE 0 END)',
@@ -34,11 +36,18 @@ class ServicegroupsummaryQuery extends IdoQuery
     );
 
     /**
-     * Subquery used for the summary query
+     * The union
      *
-     * @var IdoQuery
+     * @var Zend_Db_Select
      */
-    protected $subQuery;
+    protected $summaryQuery;
+
+    /**
+     * Subqueries used for the summary query
+     *
+     * @var IdoQuery[]
+     */
+    protected $subQueries = [];
 
     /**
      * Count query
@@ -49,7 +58,9 @@ class ServicegroupsummaryQuery extends IdoQuery
 
     public function addFilter(Filter $filter)
     {
-        $this->subQuery->applyFilter(clone $filter);
+        foreach ($this->subQueries as $sub) {
+            $sub->applyFilter(clone $filter);
+        }
         $this->countQuery->applyFilter(clone $filter);
         return $this;
     }
@@ -70,10 +81,24 @@ class ServicegroupsummaryQuery extends IdoQuery
                 'service_state'
             )
         );
-        $subQuery->setIsSubQuery();
+        $this->subQueries[] = $subQuery;
-        $this->subQuery = $subQuery;
+        $emptyGroups = $this->createSubQuery(
-        $this->select->from(array('servicesgroupsummary' => $this->subQuery), array());
+            'Emptyservicegroup',
-        $this->group(array('servicegroup_name', 'servicegroup_alias'));
+            [
+                'servicegroup_alias',
+                'servicegroup_name',
+                'service_handled'   => new Zend_Db_Expr('NULL'),
+                'service_severity'  => new Zend_Db_Expr('0'),
+                'service_state'     => new Zend_Db_Expr('NULL'),
+            ]
+        );
+        $this->subQueries[] = $emptyGroups;
+        $this->summaryQuery = $this->db->select()->union(
+            [$subQuery, $emptyGroups],
+            Zend_Db_Select::SQL_UNION_ALL
+        );
+        $this->select->from(['servicesgroupsummary' => $this->summaryQuery], []);
+        $this->group(['servicegroup_name', 'servicegroup_alias']);
         $this->joinedVirtualTables['servicegroupsummary'] = true;
     }