Merge pull request #1642 from Icinga/feature/group-resolve-housekeeping

Add group apply housekeeping and testing

application/clicommands/HousekeepingCommand.php

@@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Clicommands;
 use Icinga\Exception\MissingParameterException;
 use Icinga\Module\Director\Cli\Command;
 use Icinga\Module\Director\Db\Housekeeping;
+use Icinga\Module\Director\Db\MembershipHousekeeping;
 
 class HousekeepingCommand extends Command
 {

library/Director/Db/HostMembershipHousekeeping.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace Icinga\Module\Director\Db;
+
+class HostMembershipHousekeeping extends MembershipHousekeeping
+{
+    protected $type = 'host';
+}

library/Director/Db/Housekeeping.php

@@ -51,6 +51,7 @@ class Housekeeping
             'unlinkedImportedRowSets'    => N_('Unlinked imported row sets'),
             'unlinkedImportedRows'       => N_('Unlinked imported rows'),
             'unlinkedImportedProperties' => N_('Unlinked imported properties'),
+            'resolveCache'               => N_('(Host) group resolve cache'),
         );
     }
 
@@ -189,4 +190,16 @@ class Housekeeping
 
         return $this->db->exec($sql);
     }
+
+    public function countResolveCache()
+    {
+        $helper = MembershipHousekeeping::instance('host', $this->connection);
+        return array_sum($helper->check());
+    }
+
+    public function wipeResolveCache()
+    {
+        $helper = MembershipHousekeeping::instance('host', $this->connection);
+        return $helper->update();
+    }
 }

library/Director/Db/MembershipHousekeeping.php

@@ -0,0 +1,135 @@
+<?php
+
+namespace Icinga\Module\Director\Db;
+
+use Icinga\Module\Director\Application\MemoryLimit;
+use Icinga\Module\Director\Data\Db\DbConnection;
+use Icinga\Module\Director\Db;
+use Icinga\Module\Director\Db\Cache\PrefetchCache;
+use Icinga\Module\Director\Objects\GroupMembershipResolver;
+use Icinga\Module\Director\Objects\IcingaObject;
+use Icinga\Module\Director\Objects\IcingaObjectGroup;
+
+abstract class MembershipHousekeeping
+{
+    protected $type;
+
+    protected $groupType;
+
+    protected $connection;
+
+    /** @var GroupMembershipResolver */
+    protected $resolver;
+
+    /** @var IcingaObject[] */
+    protected $objects;
+
+    /** @var IcingaObjectGroup[] */
+    protected $groups;
+
+    protected $prepared = false;
+
+    protected static $instances = [];
+
+    public function __construct(Db $connection)
+    {
+        $this->connection = $connection;
+
+        if ($this->groupType === null) {
+            $this->groupType = $this->type . 'Group';
+        }
+    }
+
+    /**
+     * @param string       $type
+     * @param DbConnection $connection
+     *
+     * @return static
+     */
+    public static function instance($type, $connection)
+    {
+        if (! array_key_exists($type, self::$instances)) {
+            /** @var MembershipHousekeeping $class */
+            $class = 'Icinga\\Module\\Director\\Db\\' . ucfirst($type) . 'MembershipHousekeeping';
+
+            /** @var MembershipHousekeeping $helper */
+            self::$instances[$type] = new $class($connection);
+        }
+
+        return self::$instances[$type];
+    }
+
+    protected function prepare()
+    {
+        if ($this->prepared) {
+            return $this;
+        }
+
+        $this->prepareCache();
+        $this->resolver()->defer();
+
+        $this->objects = IcingaObject::loadAllByType($this->type, $this->connection);
+        $this->resolver()->addObjects($this->objects);
+
+        $this->groups = IcingaObject::loadAllByType($this->groupType, $this->connection);
+        $this->resolver()->addGroups($this->groups);
+
+        MemoryLimit::raiseTo('1024M');
+
+        $this->prepared = true;
+
+        return $this;
+    }
+
+    public function check()
+    {
+        $this->prepare();
+
+        $resolver = $this->resolver()->checkDb();
+
+        return array($resolver->getNewMappings(), $resolver->getOutdatedMappings());
+    }
+
+    public function update()
+    {
+        $this->prepare();
+
+        $this->resolver()->refreshDb(true);
+
+        return true;
+    }
+
+    protected function prepareCache()
+    {
+        PrefetchCache::initialize($this->connection);
+
+        IcingaObject::prefetchAllRelationsByType($this->type, $this->connection);
+    }
+
+    protected function resolver()
+    {
+        if ($this->resolver === null) {
+            /** @var GroupMembershipResolver $class */
+            $class = 'Icinga\\Module\\Director\\Objects\\' . ucfirst($this->type) . 'GroupMembershipResolver';
+            $this->resolver = new $class($this->connection);
+        }
+
+        return $this->resolver;
+    }
+
+    /**
+     * @return IcingaObject[]
+     */
+    public function getObjects()
+    {
+        return $this->objects;
+    }
+
+    /**
+     * @return IcingaObjectGroup[]
+     */
+    public function getGroups()
+    {
+        return $this->groups;
+    }
+}

library/Director/Objects/GroupMembershipResolver.php

@@ -47,6 +47,9 @@ abstract class GroupMembershipResolver
     /** @var bool */
     protected $deferred = false;
 
+    /** @var bool */
+    protected $checked = false;
+
     /** @var bool */
     protected $useTransactions = false;
 
@@ -67,6 +70,33 @@ abstract class GroupMembershipResolver
         return $this->clearGroups()->clearObjects()->refreshDb(true);
     }
 
+    public function checkDb()
+    {
+        if ($this->checked) {
+            return $this;
+        }
+
+        if ($this->isDeferred()) {
+            // ensure we are not working with cached data
+            IcingaTemplateRepository::clear();
+        }
+
+        Benchmark::measure('Rechecking all objects');
+        $this->recheckAllObjects($this->getAppliedGroups());
+        if (empty($this->objects)) {
+            Benchmark::measure('Nothing to check, got no qualified object');
+
+            return $this;
+        }
+
+        Benchmark::measure('Recheck done, loading existing mappings');
+        $this->fetchStoredMappings();
+        Benchmark::measure('Got stored group mappings');
+
+        $this->checked = true;
+        return $this;
+    }
+
     /**
      * @param bool $force
      * @return $this
@@ -75,23 +105,18 @@ abstract class GroupMembershipResolver
     public function refreshDb($force = false)
     {
         if ($force || ! $this->isDeferred()) {
-            if ($this->isDeferred()) {
+            $this->checkDb();
-                // ensure we are not working with cached data
-                IcingaTemplateRepository::clear();
-            }
 
-            Benchmark::measure('Rechecking all objects');
-            $this->recheckAllObjects($this->getAppliedGroups());
             if (empty($this->objects)) {
                 Benchmark::measure('Nothing to check, got no qualified object');
 
                 return $this;
             }
-            Benchmark::measure('Recheck done, loading existing mappings');
+
-            $this->fetchStoredMappings();
             Benchmark::measure('Ready, going to store new mappings');
             $this->storeNewMappings();
             $this->removeOutdatedMappings();
+            Benchmark::measure('Updated group mappings in db');
         }
 
         return $this;
@@ -235,6 +260,8 @@ abstract class GroupMembershipResolver
         $this->assertBeenLoadedFromDb($group);
         $this->groups[$group->get('id')] = $group;
 
+        $this->checked = false;
+
         return $this;
     }
 
@@ -248,6 +275,8 @@ abstract class GroupMembershipResolver
             $this->addGroup($group);
         }
 
+        $this->checked = false;
+
         return $this;
     }
 
@@ -277,15 +306,25 @@ abstract class GroupMembershipResolver
     public function clearGroups()
     {
         $this->objects = array();
+        $this->checked = false;
         return $this;
     }
 
+    public function getNewMappings()
+    {
+        if ($this->newMappings !== null && $this->existingMappings !== null) {
+            return $this->getDifference($this->newMappings, $this->existingMappings);
+        } else {
+            return [];
+        }
+    }
+
     /**
      * @throws \Zend_Db_Adapter_Exception
      */
     protected function storeNewMappings()
     {
-        $diff = $this->getDifference($this->newMappings, $this->existingMappings);
+        $diff = $this->getNewMappings();
         $count = count($diff);
         if ($count === 0) {
             return;
@@ -327,9 +366,18 @@ abstract class GroupMembershipResolver
         }
     }
 
+    public function getOutdatedMappings()
+    {
+        if ($this->newMappings !== null && $this->existingMappings !== null) {
+            return $this->getDifference($this->existingMappings, $this->newMappings);
+        } else {
+            return [];
+        }
+    }
+
     protected function removeOutdatedMappings()
     {
-        $diff = $this->getDifference($this->existingMappings, $this->newMappings);
+        $diff = $this->getOutdatedMappings();
         $count = count($diff);
         if ($count === 0) {
             return;

library/Director/Test/BaseTestCase.php

@@ -6,7 +6,6 @@ use Icinga\Application\Icinga;
 use Icinga\Application\Config;
 use Icinga\Data\ResourceFactory;
 use Icinga\Exception\ConfigurationError;
-use Icinga\Module\Director\Data\Db\DbConnection;
 use Icinga\Module\Director\Db;
 use Icinga\Module\Director\Db\Migrations;
 use Icinga\Module\Director\Objects\IcingaObject;
@@ -16,7 +15,8 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
 {
     private static $app;
 
-    private $db;
+    /** @var Db */
+    private static $db;
 
     public function setUp()
     {
@@ -39,7 +39,7 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
         return $this->getDbResourceName() !== null;
     }
 
-    protected function getDbResourceName()
+    protected static function getDbResourceName()
     {
         if (array_key_exists('DIRECTOR_TESTDB_RES', $_SERVER)) {
             return $_SERVER['DIRECTOR_TESTDB_RES'];
@@ -52,10 +52,10 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
      * @return Db
      * @throws ConfigurationError
      */
-    protected function getDb()
+    protected static function getDb()
     {
-        if ($this->db === null) {
+        if (self::$db === null) {
-            $resourceName = $this->getDbResourceName();
+            $resourceName = self::getDbResourceName();
             if (! $resourceName) {
                 throw new ConfigurationError(
                     'Could not run DB-based tests, please configure a testing db resource'
@@ -74,12 +74,12 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
             if (array_key_exists('DIRECTOR_TESTDB_PASSWORD', $_SERVER)) {
                 $dbConfig->password = $_SERVER['DIRECTOR_TESTDB_PASSWORD'];
             }
-            $this->db = new Db($dbConfig);
+            self::$db = new Db($dbConfig);
-            $migrations = new Migrations($this->db);
+            $migrations = new Migrations(self::$db);
             $migrations->applyPendingMigrations();
         }
 
-        return $this->db;
+        return self::$db;
     }
 
     protected function newObject($type, $name, $properties = array())

test/php/library/Director/Objects/HostGroupMembershipResolverTest.php

@@ -0,0 +1,356 @@
+<?php
+
+namespace Tests\Icinga\Module\Director\Objects;
+
+use Icinga\Exception\NotFoundError;
+use Icinga\Module\Director\Exception\DuplicateKeyException;
+use Icinga\Module\Director\Objects\HostGroupMembershipResolver;
+use Icinga\Module\Director\Objects\IcingaObject;
+use Icinga\Module\Director\Repository\IcingaTemplateRepository;
+use Icinga\Module\Director\Test\BaseTestCase;
+
+class HostGroupMembershipResolverTest extends BaseTestCase
+{
+    const PREFIX = '__groupmembership';
+    const TYPE = 'host';
+
+    public function setUp()
+    {
+        IcingaTemplateRepository::clear();
+    }
+
+    public static function cleanArtifacts()
+    {
+        $db = static::getDb()->getDbAdapter();
+
+        $where = sprintf("object_name LIKE '%s%%'", self::PREFIX);
+
+        $db->delete('icinga_' . self::TYPE . 'group', $where);
+
+        $db->delete('icinga_' . self::TYPE, $where . " AND object_type = 'object'");
+        $db->delete('icinga_' . self::TYPE, $where);
+    }
+
+    public static function setUpBeforeClass()
+    {
+        static::cleanArtifacts();
+    }
+
+    public static function tearDownAfterClass()
+    {
+        static::cleanArtifacts();
+    }
+
+    /**
+     * @param string $type
+     * @param string $name
+     * @param array  $props
+     *
+     * @return IcingaObject
+     * @throws DuplicateKeyException
+     * @throws \Icinga\Exception\ConfigurationError
+     */
+    protected function object($type, $name, $props = [])
+    {
+        $db = $this->getDb();
+        $fullName = self::PREFIX . $name;
+        $object = null;
+
+        try {
+            $object = IcingaObject::loadByType($type, $fullName, $db);
+
+            foreach ($props as $k => $v) {
+                $object->set($k, $v);
+            }
+
+            $object->store();
+        } catch (NotFoundError $e) {
+            $object = null;
+        }
+
+        if ($object === null) {
+            $object = IcingaObject::createByType($type, array_merge([
+                'object_name' => $fullName,
+                'object_type' => 'object',
+            ], $props), $this->getDb());
+
+            $object->store();
+        }
+
+        return $object;
+    }
+
+    protected function objects($type)
+    {
+        /** @var IcingaObject $class */
+        $class = IcingaObject::classByType($type);
+
+        /** @var IcingaObject $dummy */
+        $dummy = $class::create();
+
+        $table = $dummy->getTableName();
+        $query = $this->getDb()->getDbAdapter()->select()
+            ->from($table)
+            ->where('object_name LIKE ?', self::PREFIX . '%');
+
+        $objects = [];
+        $l = strlen(self::PREFIX);
+
+        foreach ($class::loadAll($this->getDb(), $query) as $object) {
+            $key = substr($object->getObjectName(), $l);
+            $objects[$key] = $object;
+        }
+
+        return $objects;
+    }
+
+    protected function resolved()
+    {
+        $db = $this->getDb()->getDbAdapter();
+
+        $select = $db->select()
+            ->from(
+                ['r' => 'icinga_' . self::TYPE . 'group_' . self::TYPE . '_resolved'],
+                []
+            )->join(
+                ['o' => 'icinga_' . self::TYPE],
+                'o.id = r.' . self::TYPE . '_id',
+                ['object' => 'object_name']
+            )->join(
+                ['g' => 'icinga_' . self::TYPE . 'group'],
+                'g.id = r.' . self::TYPE . 'group_id',
+                ['groupname' => 'object_name']
+            );
+
+        $map = [];
+        $l = strlen(self::PREFIX);
+
+        foreach ($db->fetchAll($select) as $row) {
+            $o = $row->object;
+            $g = $row->groupname;
+
+            if (! substr($o, 0, $l) === self::PREFIX) {
+                continue;
+            }
+            $o = substr($o, $l);
+
+            if (! substr($g, 0, $l) === self::PREFIX) {
+                continue;
+            }
+            $g = substr($g, $l);
+
+            if (! array_key_exists($o, $map)) {
+                $map[$o] = [];
+            }
+
+            $map[$o][] = $g;
+        }
+
+        return $map;
+    }
+
+    /**
+     * Creates:
+     *
+     * - 1 template
+     * - 10 hosts importing the template with a var match_var=magic
+     *
+     * @throws DuplicateKeyException
+     * @throws \Icinga\Exception\ConfigurationError
+     */
+    public function testCreateHosts()
+    {
+        // template that sets a group later
+        $template = $this->object('host', 'template', [
+            'object_type' => 'template',
+        ]);
+        $this->assertTrue($template->hasBeenLoadedFromDb());
+
+        // hosts to assign groups
+        for ($i = 1; $i <= 10; $i++) {
+            $host = $this->object('host', $i, [
+                'imports' => self::PREFIX . 'template',
+                'vars.match_var' => 'magic'
+            ]);
+            $this->assertTrue($host->hasBeenLoadedFromDb());
+        }
+    }
+
+    /**
+     * Creates:
+     *
+     * - 10 hostgroups applying on hosts with match_var=magic
+     * - 2 static hostgroups
+     *
+     * @throws DuplicateKeyException
+     * @throws \Icinga\Exception\ConfigurationError
+     */
+    public function testCreateHostgroups()
+    {
+        $filter = 'host.vars.match_var=%22magic%22';
+        for ($i = 1; $i <= 10; $i++) {
+            $hostgroup = $this->object('hostgroup', 'apply' . $i, [
+                'assign_filter' => $filter
+            ]);
+            $this->assertTrue($hostgroup->hasBeenLoadedFromDb());
+        }
+
+        // static groups
+        for ($i = 1; $i <= 2; $i++) {
+            $hostgroup = $this->object('hostgroup', 'static' . $i);
+            $this->assertTrue($hostgroup->hasBeenLoadedFromDb());
+        }
+    }
+
+    /**
+     * Assigns static groups to:
+     *
+     * - the template
+     * - the first host
+     *
+     * @throws DuplicateKeyException
+     * @throws \Icinga\Exception\ConfigurationError
+     *
+     * @depends testCreateHosts
+     * @depends testCreateHostgroups
+     */
+    public function testAddStaticGroups()
+    {
+        // add group to template
+        $template = $this->object('host', 'template');
+        $template->setGroups(self::PREFIX . 'static1');
+        $template->store();
+        $this->assertFalse($template->hasBeenModified());
+
+        // add group to first host
+        $host = $this->object('host', 1);
+        $host->setGroups(self::PREFIX . 'static2');
+        $host->store();
+        $this->assertFalse($host->hasBeenModified());
+    }
+
+    /**
+     * Asserts that static groups are resolved for hosts:
+     *
+     * - all but first should have static1
+     * - first should have static2
+     *
+     * @depends testAddStaticGroups
+     */
+    public function testStaticResolvedMappings()
+    {
+        $resolved = $this->resolved();
+
+        $this->assertArrayHasKey(
+            1,
+            $resolved,
+            'Host 1 must have groups resolved'
+        );
+
+        $this->assertContains(
+            'static2',
+            $resolved[1],
+            'Host template must have static group 1'
+        );
+
+        $hosts = $this->objects('host');
+        $this->assertNotEmpty($hosts, 'Must have hosts found in DB');
+
+        foreach ($hosts as $name => $host) {
+            if ($host->object_type === 'template') {
+                continue;
+            }
+
+            $this->assertArrayHasKey(
+                $name,
+                $resolved,
+                'All hosts must have groups resolved'
+            );
+
+            if ($name === 1) {
+                $this->assertNotContains(
+                    'static1',
+                    $resolved[$name],
+                    'First host must not have static group 1'
+                );
+            } else {
+                $this->assertContains(
+                    'static1',
+                    $resolved[$name],
+                    'All hosts but the first must have static group 1'
+                );
+            }
+        }
+    }
+
+    /**
+     * @depends testCreateHostgroups
+     */
+    public function testApplyResolvedMappings()
+    {
+        $resolved = $this->resolved();
+
+        $hosts = $this->objects('host');
+        $this->assertNotEmpty($hosts, 'Must have hosts found in DB');
+
+        foreach ($hosts as $name => $host) {
+            if ($host->object_type === 'template') {
+                continue;
+            }
+
+            $this->assertArrayHasKey($name, $resolved, 'Host must have groups resolved');
+
+            for ($i = 1; $i <= 10; $i++) {
+                $this->assertContains(
+                    'apply' . $i,
+                    $resolved[$name],
+                    'All Host objects must have all applied groups'
+                );
+            }
+        }
+    }
+
+    /**
+     * @throws DuplicateKeyException
+     * @throws \Icinga\Exception\ConfigurationError
+     *
+     * @depends testAddStaticGroups
+     */
+    public function testChangeAppliedGroupsAfterStatic()
+    {
+        $filter = 'host.vars.match_var=%22magic*%22';
+
+        $hostgroup = $this->object('hostgroup', 'apply1', [
+            'assign_filter' => $filter
+        ]);
+        $this->assertTrue($hostgroup->hasBeenLoadedFromDb());
+        $this->assertFalse($hostgroup->hasBeenModified());
+
+        $resolved = $this->resolved();
+
+        for ($i = 1; $i <= 10; $i++) {
+            $this->assertContains(
+                'apply1',
+                $resolved[$i],
+                'All Host objects must have apply1 group'
+            );
+        }
+    }
+
+    /**
+     * @throws \Icinga\Exception\ConfigurationError
+     * @throws \Zend_Db_Adapter_Exception
+     *
+     * @depends testStaticResolvedMappings
+     * @depends testApplyResolvedMappings
+     * @depends testChangeAppliedGroupsAfterStatic
+     */
+    public function testFullRecheck()
+    {
+        $resolver = new HostGroupMembershipResolver($this->getDb());
+
+        $resolver->checkDb();
+        $this->assertEmpty($resolver->getNewMappings(), 'There should not be any new mappings');
+        $this->assertEmpty($resolver->getOutdatedMappings(), 'There should not be any outdated mappings');
+    }
+}