tyler.durden
/
icingaweb2-module-director
mirror of https://github.com/Icinga/icingaweb2-module-director.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PrefetchCache: lot's of improvements 

This commit enables the prefetch cache per default when rendering configs
and adds the following features:

* prefetch all kinds of Icinga object inheritance (imports)
* prefetch group memberships
* prefetch custom variables
* render vars with the same checksum only once

Roughly measured performance boost with a large real-life config, renders three
times faster as it did before. Might be even more for those suffering from high
latencies when accessing their DB remotely and/or on a slow storage.

fixes #12876
This commit is contained in:
Thomas Gelf 2016-10-09 12:02:22 +00:00
parent 42aef0105d
commit 6f12663756
9 changed files with 226 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
library/Director/CustomVariable/CustomVariable.php
View File

@ -3,6 +3,7 @@
namespace Icinga\Module\Director\CustomVariable;
use Icinga\Exception\ProgrammingError;
use Icinga\Module\Director\Db\Cache\PrefetchCache;
use Icinga\Module\Director\IcingaConfig\IcingaConfigRenderer;
abstract class CustomVariable implements IcingaConfigRenderer
@ -21,6 +22,8 @@ abstract class CustomVariable implements IcingaConfigRenderer
protected $deleted = false;
protected $checksum;
protected function __construct($key, $value = null)
{
$this->key = $key;
@ -85,6 +88,15 @@ abstract class CustomVariable implements IcingaConfigRenderer
return $this->modified;
}
public function toConfigStringPrefetchable()
{
if (PrefetchCache::shouldBeUsed()) {
return PrefetchCache::instance()->renderVar($this);
} else {
return $this->toConfigString();
}
}
public function setModified($modified = true)
{
$this->modified = $modified;
@ -117,6 +129,17 @@ abstract class CustomVariable implements IcingaConfigRenderer
return ! $this->equals($var);
}
protected function setChecksum($checksum)
{
$this->checksum = $checksum;
return $this;
}
public function getChecksum()
{
return $this->checksum;
}
public static function wantCustomVariable($key, $value)
{
if ($value instanceof CustomVariable) {
@ -182,6 +205,9 @@ abstract class CustomVariable implements IcingaConfigRenderer
$row->format
);
}
if (property_exists($row, 'checksum')) {
$var->setChecksum($row->checksum);
}
$var->loadedFromDb = true;
$var->setUnmodified();

2
library/Director/CustomVariable/CustomVariables.php
View File

@ -249,7 +249,7 @@ class CustomVariables implements Iterator, Countable, IcingaConfigRenderer
{
return c::renderKeyValue(
$this->renderKeyName($key),
$var->toConfigString()
$var->toConfigStringPrefetchable()
);
}

22
library/Director/Db/Cache/CustomVariableCache.php
View File

@ -16,16 +16,24 @@ class CustomVariableCache
public function __construct(IcingaObject $object)
{
$db = $object->getConnection()->getDbAdapter();
$connection = $object->getConnection();
$db = $connection->getDbAdapter();
$columns = array(
'id' => sprintf('v.%s', $object->getVarsIdColumn()),
'varname' => 'v.varname',
'varvalue' => 'v.varvalue',
'format' => 'v.format',
'checksum' => '(NULL)',
);
if (! $connection->isPgsql()) {
$columns['checksum'] = "UNHEX(SHA1(v.varvalue || ';' || v.format))";
}
$query = $db->select()->from(
array('v' => $object->getVarsTableName()),
array(
'id' => sprintf('v.%s', $object->getVarsIdColumn()),
'varname' => 'v.varname',
'varvalue' => 'v.varvalue',
'format' => 'v.format',
)
$columns
);
foreach ($db->fetchAll($query) as $row) {

101
library/Director/Db/Cache/GroupMembershipCache.php Normal file
View File

@ -0,0 +1,101 @@
<?php
namespace Icinga\Module\Director\Db\Cache;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Objects\IcingaObject;
class GroupMembershipCache
{
protected $type;
protected $table;
protected $groupsClass;
protected $memberships;
/** @var Db Director database connection */
protected $connection;
public function __construct(IcingaObject $object)
{
$this->table = $object->getTableName();
$this->type = $object->getShortTableName();
$this->groupClass = 'Icinga\\Module\\Director\\Objects\\Icinga'
. ucfirst($this->type) . 'Group';
$this->connection = $object->getConnection();
$this->loadAllMemberships();
}
protected function loadAllMemberships()
{
$db = $this->connection->getDbAdapter();
$this->memberships = array();
$type = $this->type;
$table = $this->table;
$query = $db->select()->from(
array('o' => $table),
array(
'object_id' => 'o.id',
'group_id' => 'g.id',
'group_name' => 'g.object_name',
)
)->join(
array('go' => $table . 'group_' . $type),
'o.id = go.' . $type . '_id',
array()
)->join(
array('g' => $table . 'group'),
'go.' . $type . 'group_id = g.id',
array()
)->order('g.object_name');
foreach ($db->fetchAll($query) as $row) {
if (! array_key_exists($row->object_id, $this->membershipsById)) {
$this->membershipsById[$row->object_id] = array();
}
$this->membershipsById[$row->object_id][$row->group_id] = $row->group_name;
}
}
public function listGroupNamesForObject(IcingaObject $object)
{
if (array_key_exists($object->id, $this->memberships)) {
return array_values($this->memberships[$object->id]);
}
return array();
}
public function listGroupIdsForObject(IcingaObject $object)
{
if (array_key_exists($object->id, $this->memberships)) {
return array_keys($this->memberships[$object->id]);
}
return array();
}
public function getGroupsForObject(IcingaObject $object)
{
$groups = array();
$class = $this->groupsClass;
foreach ($this->listGroupIdsForObject($object) as $id) {
$groups[] = $class::loadWithAutoIncId($id, $this->connection);
}
return $groups;
}
public function __destruct()
{
unset($this->connection);
}
}

50
library/Director/Db/Cache/PrefetchCache.php
View File

@ -3,21 +3,32 @@
namespace Icinga\Module\Director\Db\Cache;
use Icinga\Exception\ProgrammingError;
use Icinga\Module\Director\CustomVariable\CustomVariable;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Objects\IcingaObject;
use Icinga\Module\Director\Objects\IcingaTemplateResolver;
/**
* Central prefetch cache
*
* Might be improved, accept various caches based on an interface and then
* finally replace prefetch logic in DbObject itself. This would also allow
* to get rid of IcingaObject-related code in this place
*/
class PrefetchCache
{
protected $db;
protected static $instance;
protected $caches = array();
protected $varsCaches = array();
protected $groupsCaches = array();
protected $templateResolvers = array();
protected $renderedVars = array();
public static function initialize(Db $db)
{
self::$instance = new static($db);
@ -57,6 +68,12 @@ class PrefetchCache
return $this->groupsCache($object)->getGroupsForObject($object);
}
public function imports(IcingaObject $object)
{
return $this->templateResolver($object)->setObject($object)->fetchParents();
}
/* Hint: not implemented, this happens in DbObject right now
public function byObjectType($type)
{
if (! array_key_exists($type, $this->caches)) {
@ -65,6 +82,21 @@ class PrefetchCache
return $this->caches[$type];
}
*/
public function renderVar(CustomVariable $var)
{
$checksum = $var->getChecksum();
if (null === $checksum) {
return $var->toConfigString();
} else {
if (! array_key_exists($checksum, $this->renderedVars)) {
$this->renderedVars[$checksum] = $var->toConfigString();
}
return $this->renderedVars[$checksum];
}
}
protected function varsCache(IcingaObject $object)
{
@ -77,6 +109,17 @@ class PrefetchCache
return $this->varsCaches[$key];
}
protected function templateResolver(IcingaObject $object)
{
$key = $object->getShortTableName();
if (! array_key_exists($key, $this->templateResolvers)) {
$this->templateResolvers[$key] = new IcingaTemplateResolver($object);
}
return $this->templateResolvers[$key];
}
protected function groupsCache(IcingaObject $object)
{
$key = $object->getShortTableName();
@ -90,8 +133,9 @@ class PrefetchCache
public function __destruct()
{
unset($this->caches);
unset($this->groupsCaches);
unset($this->varsCaches);
unset($this->templateResolvers);
unset($this->renderedVars);
}
}

7
library/Director/Objects/IcingaObject.php
View File

@ -4,6 +4,7 @@ namespace Icinga\Module\Director\Objects;
use Icinga\Module\Director\CustomVariable\CustomVariables;
use Icinga\Module\Director\Data\Db\DbObject;
use Icinga\Module\Director\Db\Cache\PrefetchCache;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\IcingaConfig\IcingaConfig;
use Icinga\Module\Director\IcingaConfig\IcingaConfigRenderer;
@ -998,7 +999,11 @@ abstract class IcingaObject extends DbObject implements IcingaConfigRenderer
$this->assertCustomVarsSupport();
if ($this->vars === null) {
if ($this->hasBeenLoadedFromDb()) {
$this->vars = CustomVariables::loadForStoredObject($this);
if (PrefetchCache::shouldBeUsed()) {
$this->vars = PrefetchCache::instance()->vars($this);
} else {
$this->vars = CustomVariables::loadForStoredObject($this);
}
} else {
$this->vars = new CustomVariables();
}

11
library/Director/Objects/IcingaObjectGroups.php
View File

@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Objects;
use Icinga\Exception\ProgrammingError;
use Iterator;
use Countable;
use Icinga\Module\Director\Db\Cache\PrefetchCache;
use Icinga\Module\Director\IcingaConfig\IcingaConfigRenderer;
use Icinga\Module\Director\IcingaConfig\IcingaConfigHelper as c;
@ -318,7 +319,15 @@ class IcingaObjectGroups implements Iterator, Countable, IcingaConfigRenderer
public static function loadForStoredObject(IcingaObject $object)
{
$groups = new static($object);
return $groups->loadFromDb();
if (PrefetchCache::shouldBeUsed()) {
$groups->groups = PrefetchCache::instance()->groups($object);
$groups->cloneStored();
} else {
$groups->loadFromDb();
}
return $groups;
}
public function toConfigString()

21
library/Director/Objects/IcingaObjectImports.php
View File

@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Objects;
use Icinga\Exception\ProgrammingError;
use Iterator;
use Countable;
use Icinga\Module\Director\Db\Cache\PrefetchCache;
use Icinga\Module\Director\IcingaConfig\IcingaConfigRenderer;
use Icinga\Module\Director\IcingaConfig\IcingaConfigHelper as c;
use Icinga\Module\Director\IcingaConfig\IcingaLegacyConfigHelper as c1;
@ -218,7 +219,7 @@ class IcingaObjectImports implements Iterator, Countable, IcingaConfigRenderer
$class = $this->getImportClass();
if (is_array($this->object->getKeyName())) {
// Services only
$import = $class::load(array('object_name' => $name), $connection);
$import = $class::load(array('object_name' => $name, 'object_type' => 'template'), $connection);
} else {
$import = $class::load($name, $connection);
}
@ -280,7 +281,17 @@ class IcingaObjectImports implements Iterator, Countable, IcingaConfigRenderer
$this->storedImports[$k] = clone($v);
}
$this->cloneStored();
return $this;
}
protected function loadFromPrefetchCache()
{
$this->storedImports = $this->objects = PrefetchCache::instance()->imports($this->object);
$this->imports = array();
foreach ($this->objects as $o) {
$this->imports[$o->object_name] = $o->object_name;
}
return $this;
}
@ -336,7 +347,11 @@ class IcingaObjectImports implements Iterator, Countable, IcingaConfigRenderer
public static function loadForStoredObject(IcingaObject $object)
{
$imports = new static($object);
return $imports->loadFromDb();
if (PrefetchCache::shouldBeUsed()) {
return $imports->loadFromPrefetchCache();
} else {
return $imports->loadFromDb();
}
}
public function toConfigString()

3
library/Director/Objects/IcingaTemplateResolver.php
View File

@ -53,7 +53,8 @@ class IcingaTemplateResolver
$res = array();
$class = $this->object;
foreach ($this->listParentIds() as $id) {
$res[] = $class::loadWithAutoIncId($id);
$object = $class::loadWithAutoIncId($id, $this->connection);
$res[$object->object_name] = $object;
}
return $res;