Merge pull request #1642 from Icinga/feature/group-resolve-housekeeping

Add group apply housekeeping and testing
This commit is contained in:
Markus Frosch 2018-09-18 17:18:16 +02:00 committed by GitHub
commit e2bd821d26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 580 additions and 19 deletions

View File

@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Clicommands;
use Icinga\Exception\MissingParameterException;
use Icinga\Module\Director\Cli\Command;
use Icinga\Module\Director\Db\Housekeeping;
use Icinga\Module\Director\Db\MembershipHousekeeping;
class HousekeepingCommand extends Command
{

View File

@ -0,0 +1,8 @@
<?php
namespace Icinga\Module\Director\Db;
class HostMembershipHousekeeping extends MembershipHousekeeping
{
protected $type = 'host';
}

View File

@ -51,6 +51,7 @@ class Housekeeping
'unlinkedImportedRowSets' => N_('Unlinked imported row sets'),
'unlinkedImportedRows' => N_('Unlinked imported rows'),
'unlinkedImportedProperties' => N_('Unlinked imported properties'),
'resolveCache' => N_('(Host) group resolve cache'),
);
}
@ -189,4 +190,16 @@ class Housekeeping
return $this->db->exec($sql);
}
public function countResolveCache()
{
$helper = MembershipHousekeeping::instance('host', $this->connection);
return array_sum($helper->check());
}
public function wipeResolveCache()
{
$helper = MembershipHousekeeping::instance('host', $this->connection);
return $helper->update();
}
}

View File

@ -0,0 +1,135 @@
<?php
namespace Icinga\Module\Director\Db;
use Icinga\Module\Director\Application\MemoryLimit;
use Icinga\Module\Director\Data\Db\DbConnection;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Db\Cache\PrefetchCache;
use Icinga\Module\Director\Objects\GroupMembershipResolver;
use Icinga\Module\Director\Objects\IcingaObject;
use Icinga\Module\Director\Objects\IcingaObjectGroup;
abstract class MembershipHousekeeping
{
protected $type;
protected $groupType;
protected $connection;
/** @var GroupMembershipResolver */
protected $resolver;
/** @var IcingaObject[] */
protected $objects;
/** @var IcingaObjectGroup[] */
protected $groups;
protected $prepared = false;
protected static $instances = [];
public function __construct(Db $connection)
{
$this->connection = $connection;
if ($this->groupType === null) {
$this->groupType = $this->type . 'Group';
}
}
/**
* @param string $type
* @param DbConnection $connection
*
* @return static
*/
public static function instance($type, $connection)
{
if (! array_key_exists($type, self::$instances)) {
/** @var MembershipHousekeeping $class */
$class = 'Icinga\\Module\\Director\\Db\\' . ucfirst($type) . 'MembershipHousekeeping';
/** @var MembershipHousekeeping $helper */
self::$instances[$type] = new $class($connection);
}
return self::$instances[$type];
}
protected function prepare()
{
if ($this->prepared) {
return $this;
}
$this->prepareCache();
$this->resolver()->defer();
$this->objects = IcingaObject::loadAllByType($this->type, $this->connection);
$this->resolver()->addObjects($this->objects);
$this->groups = IcingaObject::loadAllByType($this->groupType, $this->connection);
$this->resolver()->addGroups($this->groups);
MemoryLimit::raiseTo('1024M');
$this->prepared = true;
return $this;
}
public function check()
{
$this->prepare();
$resolver = $this->resolver()->checkDb();
return array($resolver->getNewMappings(), $resolver->getOutdatedMappings());
}
public function update()
{
$this->prepare();
$this->resolver()->refreshDb(true);
return true;
}
protected function prepareCache()
{
PrefetchCache::initialize($this->connection);
IcingaObject::prefetchAllRelationsByType($this->type, $this->connection);
}
protected function resolver()
{
if ($this->resolver === null) {
/** @var GroupMembershipResolver $class */
$class = 'Icinga\\Module\\Director\\Objects\\' . ucfirst($this->type) . 'GroupMembershipResolver';
$this->resolver = new $class($this->connection);
}
return $this->resolver;
}
/**
* @return IcingaObject[]
*/
public function getObjects()
{
return $this->objects;
}
/**
* @return IcingaObjectGroup[]
*/
public function getGroups()
{
return $this->groups;
}
}

View File

@ -47,6 +47,9 @@ abstract class GroupMembershipResolver
/** @var bool */
protected $deferred = false;
/** @var bool */
protected $checked = false;
/** @var bool */
protected $useTransactions = false;
@ -67,6 +70,33 @@ abstract class GroupMembershipResolver
return $this->clearGroups()->clearObjects()->refreshDb(true);
}
public function checkDb()
{
if ($this->checked) {
return $this;
}
if ($this->isDeferred()) {
// ensure we are not working with cached data
IcingaTemplateRepository::clear();
}
Benchmark::measure('Rechecking all objects');
$this->recheckAllObjects($this->getAppliedGroups());
if (empty($this->objects)) {
Benchmark::measure('Nothing to check, got no qualified object');
return $this;
}
Benchmark::measure('Recheck done, loading existing mappings');
$this->fetchStoredMappings();
Benchmark::measure('Got stored group mappings');
$this->checked = true;
return $this;
}
/**
* @param bool $force
* @return $this
@ -75,23 +105,18 @@ abstract class GroupMembershipResolver
public function refreshDb($force = false)
{
if ($force || ! $this->isDeferred()) {
if ($this->isDeferred()) {
// ensure we are not working with cached data
IcingaTemplateRepository::clear();
}
$this->checkDb();
Benchmark::measure('Rechecking all objects');
$this->recheckAllObjects($this->getAppliedGroups());
if (empty($this->objects)) {
Benchmark::measure('Nothing to check, got no qualified object');
return $this;
}
Benchmark::measure('Recheck done, loading existing mappings');
$this->fetchStoredMappings();
Benchmark::measure('Ready, going to store new mappings');
$this->storeNewMappings();
$this->removeOutdatedMappings();
Benchmark::measure('Updated group mappings in db');
}
return $this;
@ -235,6 +260,8 @@ abstract class GroupMembershipResolver
$this->assertBeenLoadedFromDb($group);
$this->groups[$group->get('id')] = $group;
$this->checked = false;
return $this;
}
@ -248,6 +275,8 @@ abstract class GroupMembershipResolver
$this->addGroup($group);
}
$this->checked = false;
return $this;
}
@ -277,15 +306,25 @@ abstract class GroupMembershipResolver
public function clearGroups()
{
$this->objects = array();
$this->checked = false;
return $this;
}
public function getNewMappings()
{
if ($this->newMappings !== null && $this->existingMappings !== null) {
return $this->getDifference($this->newMappings, $this->existingMappings);
} else {
return [];
}
}
/**
* @throws \Zend_Db_Adapter_Exception
*/
protected function storeNewMappings()
{
$diff = $this->getDifference($this->newMappings, $this->existingMappings);
$diff = $this->getNewMappings();
$count = count($diff);
if ($count === 0) {
return;
@ -327,9 +366,18 @@ abstract class GroupMembershipResolver
}
}
public function getOutdatedMappings()
{
if ($this->newMappings !== null && $this->existingMappings !== null) {
return $this->getDifference($this->existingMappings, $this->newMappings);
} else {
return [];
}
}
protected function removeOutdatedMappings()
{
$diff = $this->getDifference($this->existingMappings, $this->newMappings);
$diff = $this->getOutdatedMappings();
$count = count($diff);
if ($count === 0) {
return;

View File

@ -6,7 +6,6 @@ use Icinga\Application\Icinga;
use Icinga\Application\Config;
use Icinga\Data\ResourceFactory;
use Icinga\Exception\ConfigurationError;
use Icinga\Module\Director\Data\Db\DbConnection;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Db\Migrations;
use Icinga\Module\Director\Objects\IcingaObject;
@ -16,7 +15,8 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
{
private static $app;
private $db;
/** @var Db */
private static $db;
public function setUp()
{
@ -39,7 +39,7 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
return $this->getDbResourceName() !== null;
}
protected function getDbResourceName()
protected static function getDbResourceName()
{
if (array_key_exists('DIRECTOR_TESTDB_RES', $_SERVER)) {
return $_SERVER['DIRECTOR_TESTDB_RES'];
@ -52,10 +52,10 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
* @return Db
* @throws ConfigurationError
*/
protected function getDb()
protected static function getDb()
{
if ($this->db === null) {
$resourceName = $this->getDbResourceName();
if (self::$db === null) {
$resourceName = self::getDbResourceName();
if (! $resourceName) {
throw new ConfigurationError(
'Could not run DB-based tests, please configure a testing db resource'
@ -74,12 +74,12 @@ abstract class BaseTestCase extends PHPUnit_Framework_TestCase
if (array_key_exists('DIRECTOR_TESTDB_PASSWORD', $_SERVER)) {
$dbConfig->password = $_SERVER['DIRECTOR_TESTDB_PASSWORD'];
}
$this->db = new Db($dbConfig);
$migrations = new Migrations($this->db);
self::$db = new Db($dbConfig);
$migrations = new Migrations(self::$db);
$migrations->applyPendingMigrations();
}
return $this->db;
return self::$db;
}
protected function newObject($type, $name, $properties = array())

View File

@ -0,0 +1,356 @@
<?php
namespace Tests\Icinga\Module\Director\Objects;
use Icinga\Exception\NotFoundError;
use Icinga\Module\Director\Exception\DuplicateKeyException;
use Icinga\Module\Director\Objects\HostGroupMembershipResolver;
use Icinga\Module\Director\Objects\IcingaObject;
use Icinga\Module\Director\Repository\IcingaTemplateRepository;
use Icinga\Module\Director\Test\BaseTestCase;
class HostGroupMembershipResolverTest extends BaseTestCase
{
const PREFIX = '__groupmembership';
const TYPE = 'host';
public function setUp()
{
IcingaTemplateRepository::clear();
}
public static function cleanArtifacts()
{
$db = static::getDb()->getDbAdapter();
$where = sprintf("object_name LIKE '%s%%'", self::PREFIX);
$db->delete('icinga_' . self::TYPE . 'group', $where);
$db->delete('icinga_' . self::TYPE, $where . " AND object_type = 'object'");
$db->delete('icinga_' . self::TYPE, $where);
}
public static function setUpBeforeClass()
{
static::cleanArtifacts();
}
public static function tearDownAfterClass()
{
static::cleanArtifacts();
}
/**
* @param string $type
* @param string $name
* @param array $props
*
* @return IcingaObject
* @throws DuplicateKeyException
* @throws \Icinga\Exception\ConfigurationError
*/
protected function object($type, $name, $props = [])
{
$db = $this->getDb();
$fullName = self::PREFIX . $name;
$object = null;
try {
$object = IcingaObject::loadByType($type, $fullName, $db);
foreach ($props as $k => $v) {
$object->set($k, $v);
}
$object->store();
} catch (NotFoundError $e) {
$object = null;
}
if ($object === null) {
$object = IcingaObject::createByType($type, array_merge([
'object_name' => $fullName,
'object_type' => 'object',
], $props), $this->getDb());
$object->store();
}
return $object;
}
protected function objects($type)
{
/** @var IcingaObject $class */
$class = IcingaObject::classByType($type);
/** @var IcingaObject $dummy */
$dummy = $class::create();
$table = $dummy->getTableName();
$query = $this->getDb()->getDbAdapter()->select()
->from($table)
->where('object_name LIKE ?', self::PREFIX . '%');
$objects = [];
$l = strlen(self::PREFIX);
foreach ($class::loadAll($this->getDb(), $query) as $object) {
$key = substr($object->getObjectName(), $l);
$objects[$key] = $object;
}
return $objects;
}
protected function resolved()
{
$db = $this->getDb()->getDbAdapter();
$select = $db->select()
->from(
['r' => 'icinga_' . self::TYPE . 'group_' . self::TYPE . '_resolved'],
[]
)->join(
['o' => 'icinga_' . self::TYPE],
'o.id = r.' . self::TYPE . '_id',
['object' => 'object_name']
)->join(
['g' => 'icinga_' . self::TYPE . 'group'],
'g.id = r.' . self::TYPE . 'group_id',
['groupname' => 'object_name']
);
$map = [];
$l = strlen(self::PREFIX);
foreach ($db->fetchAll($select) as $row) {
$o = $row->object;
$g = $row->groupname;
if (! substr($o, 0, $l) === self::PREFIX) {
continue;
}
$o = substr($o, $l);
if (! substr($g, 0, $l) === self::PREFIX) {
continue;
}
$g = substr($g, $l);
if (! array_key_exists($o, $map)) {
$map[$o] = [];
}
$map[$o][] = $g;
}
return $map;
}
/**
* Creates:
*
* - 1 template
* - 10 hosts importing the template with a var match_var=magic
*
* @throws DuplicateKeyException
* @throws \Icinga\Exception\ConfigurationError
*/
public function testCreateHosts()
{
// template that sets a group later
$template = $this->object('host', 'template', [
'object_type' => 'template',
]);
$this->assertTrue($template->hasBeenLoadedFromDb());
// hosts to assign groups
for ($i = 1; $i <= 10; $i++) {
$host = $this->object('host', $i, [
'imports' => self::PREFIX . 'template',
'vars.match_var' => 'magic'
]);
$this->assertTrue($host->hasBeenLoadedFromDb());
}
}
/**
* Creates:
*
* - 10 hostgroups applying on hosts with match_var=magic
* - 2 static hostgroups
*
* @throws DuplicateKeyException
* @throws \Icinga\Exception\ConfigurationError
*/
public function testCreateHostgroups()
{
$filter = 'host.vars.match_var=%22magic%22';
for ($i = 1; $i <= 10; $i++) {
$hostgroup = $this->object('hostgroup', 'apply' . $i, [
'assign_filter' => $filter
]);
$this->assertTrue($hostgroup->hasBeenLoadedFromDb());
}
// static groups
for ($i = 1; $i <= 2; $i++) {
$hostgroup = $this->object('hostgroup', 'static' . $i);
$this->assertTrue($hostgroup->hasBeenLoadedFromDb());
}
}
/**
* Assigns static groups to:
*
* - the template
* - the first host
*
* @throws DuplicateKeyException
* @throws \Icinga\Exception\ConfigurationError
*
* @depends testCreateHosts
* @depends testCreateHostgroups
*/
public function testAddStaticGroups()
{
// add group to template
$template = $this->object('host', 'template');
$template->setGroups(self::PREFIX . 'static1');
$template->store();
$this->assertFalse($template->hasBeenModified());
// add group to first host
$host = $this->object('host', 1);
$host->setGroups(self::PREFIX . 'static2');
$host->store();
$this->assertFalse($host->hasBeenModified());
}
/**
* Asserts that static groups are resolved for hosts:
*
* - all but first should have static1
* - first should have static2
*
* @depends testAddStaticGroups
*/
public function testStaticResolvedMappings()
{
$resolved = $this->resolved();
$this->assertArrayHasKey(
1,
$resolved,
'Host 1 must have groups resolved'
);
$this->assertContains(
'static2',
$resolved[1],
'Host template must have static group 1'
);
$hosts = $this->objects('host');
$this->assertNotEmpty($hosts, 'Must have hosts found in DB');
foreach ($hosts as $name => $host) {
if ($host->object_type === 'template') {
continue;
}
$this->assertArrayHasKey(
$name,
$resolved,
'All hosts must have groups resolved'
);
if ($name === 1) {
$this->assertNotContains(
'static1',
$resolved[$name],
'First host must not have static group 1'
);
} else {
$this->assertContains(
'static1',
$resolved[$name],
'All hosts but the first must have static group 1'
);
}
}
}
/**
* @depends testCreateHostgroups
*/
public function testApplyResolvedMappings()
{
$resolved = $this->resolved();
$hosts = $this->objects('host');
$this->assertNotEmpty($hosts, 'Must have hosts found in DB');
foreach ($hosts as $name => $host) {
if ($host->object_type === 'template') {
continue;
}
$this->assertArrayHasKey($name, $resolved, 'Host must have groups resolved');
for ($i = 1; $i <= 10; $i++) {
$this->assertContains(
'apply' . $i,
$resolved[$name],
'All Host objects must have all applied groups'
);
}
}
}
/**
* @throws DuplicateKeyException
* @throws \Icinga\Exception\ConfigurationError
*
* @depends testAddStaticGroups
*/
public function testChangeAppliedGroupsAfterStatic()
{
$filter = 'host.vars.match_var=%22magic*%22';
$hostgroup = $this->object('hostgroup', 'apply1', [
'assign_filter' => $filter
]);
$this->assertTrue($hostgroup->hasBeenLoadedFromDb());
$this->assertFalse($hostgroup->hasBeenModified());
$resolved = $this->resolved();
for ($i = 1; $i <= 10; $i++) {
$this->assertContains(
'apply1',
$resolved[$i],
'All Host objects must have apply1 group'
);
}
}
/**
* @throws \Icinga\Exception\ConfigurationError
* @throws \Zend_Db_Adapter_Exception
*
* @depends testStaticResolvedMappings
* @depends testApplyResolvedMappings
* @depends testChangeAppliedGroupsAfterStatic
*/
public function testFullRecheck()
{
$resolver = new HostGroupMembershipResolver($this->getDb());
$resolver->checkDb();
$this->assertEmpty($resolver->getNewMappings(), 'There should not be any new mappings');
$this->assertEmpty($resolver->getOutdatedMappings(), 'There should not be any outdated mappings');
}
}