Merge branch 'ent-3961-Añadir-caché-para-los-elementos-de-la-Consola-Visual' into 'develop'

Ent 3961 añadir caché para los elementos de la consola visual

See merge request artica/pandorafms!2443
This commit is contained in:
Alejandro Fraguas 2019-05-29 15:48:45 +02:00
commit 2e3338b595
14 changed files with 489 additions and 19 deletions

View File

@ -16,10 +16,31 @@ INSERT INTO `tnotification_source_user` (`id_source`, `id_user`, `enabled`, `als
UPDATE `tnotification_source` SET `enabled`=1 WHERE `description` = 'System status' OR `description` = 'Official communication';
UPDATE `tnotification_source` SET `icon`="icono_logo_pandora.png" WHERE `description` = 'Official communication';
-- ---------------------------------------------------------------------
-- Table `tvisual_console_items_cache`
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS `tvisual_console_elements_cache` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`vc_id` INTEGER UNSIGNED NOT NULL,
`vc_item_id` INTEGER UNSIGNED NOT NULL,
`user_id` VARCHAR(60) DEFAULT NULL,
`data` TEXT NOT NULL,
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`expiration` INTEGER UNSIGNED NOT NULL COMMENT 'Seconds to expire',
PRIMARY KEY(`id`),
FOREIGN KEY(`vc_id`) REFERENCES `tlayout`(`id`)
ON DELETE CASCADE,
FOREIGN KEY(`vc_item_id`) REFERENCES `tlayout_data`(`id`)
ON DELETE CASCADE,
FOREIGN KEY (`user_id`) REFERENCES `tusuario`(`id_user`)
ON DELETE CASCADE
ON UPDATE CASCADE
) engine=InnoDB DEFAULT CHARSET=utf8;
ALTER TABLE `tlayout_data` ADD COLUMN `cache_expiration` INTEGER UNSIGNED NOT NULL DEFAULT 0;
ALTER TABLE `tusuario` ADD COLUMN `ehorus_user_level_user` VARCHAR(60);
ALTER TABLE `tusuario` ADD COLUMN `ehorus_user_level_pass` VARCHAR(45);
ALTER TABLE `tusuario` ADD COLUMN `ehorus_user_level_enabled` TINYINT(1) DEFAULT '1';
COMMIT;

View File

@ -1375,6 +1375,7 @@ ALTER TABLE `tlayout_data` ADD COLUMN `linked_layout_status_type` ENUM ('default
ALTER TABLE `tlayout_data` ADD COLUMN `linked_layout_status_as_service_warning` FLOAT(20, 3) NOT NULL default 0;
ALTER TABLE `tlayout_data` ADD COLUMN `linked_layout_status_as_service_critical` FLOAT(20, 3) NOT NULL default 0;
ALTER TABLE `tlayout_data` ADD COLUMN `linked_layout_node_id` INT(10) NOT NULL default 0;
ALTER TABLE `tlayout_data` ADD COLUMN `cache_expiration` INTEGER UNSIGNED NOT NULL DEFAULT 0;
-- ---------------------------------------------------------------------
-- Table `tagent_custom_fields`
@ -2143,3 +2144,24 @@ INSERT INTO `tnews` (`id_news`, `author`, `subject`, `text`, `timestamp`) VALUES
ALTER TABLE `talert_templates` MODIFY COLUMN `type` ENUM('regex','max_min','max','min','equal','not_equal','warning','critical','onchange','unknown','always','not_normal');
-- ---------------------------------------------------------------------
-- Table `tvisual_console_items_cache`
-- ---------------------------------------------------------------------
CREATE TABLE `tvisual_console_elements_cache` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`vc_id` INTEGER UNSIGNED NOT NULL,
`vc_item_id` INTEGER UNSIGNED NOT NULL,
`user_id` VARCHAR(60) DEFAULT NULL,
`data` TEXT NOT NULL,
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`expiration` INTEGER UNSIGNED NOT NULL COMMENT 'Seconds to expire',
PRIMARY KEY(`id`),
FOREIGN KEY(`vc_id`) REFERENCES `tlayout`(`id`)
ON DELETE CASCADE,
FOREIGN KEY(`vc_item_id`) REFERENCES `tlayout_data`(`id`)
ON DELETE CASCADE,
FOREIGN KEY (`user_id`) REFERENCES `tusuario`(`id_user`)
ON DELETE CASCADE
ON UPDATE CASCADE
) engine=InnoDB DEFAULT CHARSET=utf8;

View File

@ -1,3 +1,5 @@
/* globals $ */
// Pandora FMS - http://pandorafms.com
// ==================================================
// Copyright (c) 2005-2009 Artica Soluciones Tecnologicas
@ -31,6 +33,8 @@ var SIZE_GRID = 16; //Const the size (for width and height) of grid.
var img_handler_start;
var img_handler_end;
var default_cache_expiration = null;
function toggle_advance_options_palette(close) {
if ($("#advance_options").css("display") == "none") {
$("#advance_options").css("display", "");
@ -1183,6 +1187,10 @@ function readFields() {
values["timezone"] = $("select[name=timezone]").val();
values["clock_animation"] = $("select[name=clock_animation]").val();
values["show_last_value"] = $("select[name=last_value]").val();
values["cache_expiration"] =
typeof $("#hidden-cache_expiration").val() !== "undefined"
? $("#hidden-cache_expiration").val()
: $("#cache_expiration").val();
// Color Cloud values
if (selectedItem == "color_cloud" || creationItem == "color_cloud") {
@ -1226,7 +1234,6 @@ function readFields() {
function create_button_palette_callback() {
var values = readFields();
//VALIDATE DATA
var validate = true;
switch (creationItem) {
@ -2219,6 +2226,26 @@ function loadFieldsFromDB(item) {
});
}
if (key == "cache_expiration") {
var intoCacheExpSelect = false;
var cacheExpId = $("#hidden-cache_expiration").attr("class");
$("#" + cacheExpId + "_select option").each(function() {
if ($(this).val() == val) {
$(this).prop("selected", true);
$(this).trigger("change");
intoCacheExpSelect = true;
}
});
if (intoCacheExpSelect == false) {
$("#" + cacheExpId + "_select").val(0);
$("#" + cacheExpId + "_units").val(1);
$("#hidden-cache_expiration").val(val);
$("#text-" + cacheExpId + "_text").val(val);
$("#" + cacheExpId + "_default").hide();
$("#" + cacheExpId + "_manual").show();
}
}
if (key == "value_show") {
$("select[name=value_show]").val(val);
}
@ -2595,6 +2622,9 @@ function hiddenFields(item) {
$("#line_case").css("display", "none");
$("#line_case." + item).css("display", "");
$("#cache_expiration_row").css("display", "none");
$("#cache_expiration_row." + item).css("display", "");
// Color cloud rows
$("#color_cloud_diameter_row").hide();
$("#color_cloud_diameter_row." + item).show();
@ -2668,6 +2698,44 @@ function cleanFields(item) {
$("table.color-range-creation input[type=color]").val("#FFFFFF");
$("table.color-range:not(table.color-range-creation)").remove();
// Clean the cache expiration selection.
if (default_cache_expiration === null) {
var cacheExpVal = $("#hidden-cache_expiration").val();
if (!Number.isNaN(Number.parseInt(cacheExpVal))) {
cacheExpVal = Number.parseInt(cacheExpVal);
} else {
cacheExpVal = 0;
}
default_cache_expiration = cacheExpVal;
}
var cacheExpId = $("#hidden-cache_expiration").attr("class");
$("#hidden-cache_expiration").val(default_cache_expiration);
var intoCacheExpSelect = false;
$("#" + cacheExpId + "_select option").each(function() {
if ($(this).val() == default_cache_expiration) {
$(this).prop("selected", true);
$(this).trigger("change");
intoCacheExpSelect = true;
}
});
if (!intoCacheExpSelect) {
// Show input.
$("#" + cacheExpId + "_select").val(0);
$("#" + cacheExpId + "_units").val(1);
$("#text-" + cacheExpId + "_text").val(default_cache_expiration);
$("#" + cacheExpId + "_default").hide();
$("#" + cacheExpId + "_manual").show();
} else {
// Show select.
$("#" + cacheExpId + "_select").val(default_cache_expiration);
$("#" + cacheExpId + "_units").val(0);
$("#text-" + cacheExpId + "_text").val("");
$("#" + cacheExpId + "_default").show();
$("#" + cacheExpId + "_manual").hide();
}
$("#preview").empty();
if (item == "simple_value") {

View File

@ -918,6 +918,32 @@ $row++;
);
$row++;
$intervals = [
10 => '10 '.__('seconds'),
30 => '30 '.__('seconds'),
60 => '1 '.__('minutes'),
300 => '5 '.__('minutes'),
900 => '15 '.__('minutes'),
1800 => '30 '.__('minutes'),
3600 => '1 '.__('hour'),
];
$table_vc->data[$row][0] = __('Default cache expiration');
$table_vc->data[$row][1] = html_print_extended_select_for_time(
'vc_default_cache_expiration',
$config['vc_default_cache_expiration'],
'',
__('No cache'),
0,
false,
true,
false,
false,
'',
false,
$intervals
);
$row++;
$table_vc->data[$row][0] = __('Default interval for refresh on Visual Console').ui_print_help_tip(__('This interval will affect to Visual Console pages'), true);
$table_vc->data[$row][1] = html_print_select($values, 'vc_refr', (int) $config['vc_refr'], '', 'N/A', 0, true, false, false);
$row++;
@ -1394,6 +1420,15 @@ tinyMCE.init({
});
$(document).ready (function () {
// Show the cache expiration conf or not.
$("input[name=legacy_vc]").change(function (e) {
if ($(this).prop("checked") === true) {
$("select#vc_default_cache_expiration_select").closest("tr").hide();
} else {
$("select#vc_default_cache_expiration_select").closest("tr").show();
}
}).change();
var comfort = 0;

View File

@ -176,6 +176,7 @@ $default_color = get_parameter('default_color', '#FFFFFF');
$color_range_from_values = get_parameter('color_range_from_values', []);
$color_range_to_values = get_parameter('color_range_to_values', []);
$color_range_colors = get_parameter('color_range_colors', []);
$cache_expiration = (int) get_parameter('cache_expiration');
switch ($action) {
case 'get_font':
@ -581,7 +582,21 @@ switch ($action) {
$values['label_position'] = $label_position;
$values['show_on_top'] = $show_on_top;
// In Graphs, background color is stored in column image (sorry)
switch ($type) {
case 'line_item':
case 'box_item':
case 'clock':
case 'icon':
case 'label':
$values['cache_expiration'] = 0;
break;
default:
$values['cache_expiration'] = $cache_expiration;
break;
}
// In Graphs, background color is stored in column image (sorry).
if ($type == 'module_graph') {
$values['image'] = $background_color;
$values['type_graph'] = $type_graph;
@ -997,6 +1012,7 @@ switch ($action) {
unset($values['id_layout_linked']);
unset($values['element_group']);
unset($values['id_layout_linked_weight']);
unset($values['cache_expiration']);
// Don't change background color in graphs when move
switch ($type) {
case 'group_item':
@ -1070,6 +1086,16 @@ switch ($action) {
['id' => $id_element]
);
// Invalidate the item's cache.
if ($result !== false && $result > 0) {
db_process_sql_delete(
'tvisual_console_elements_cache',
[
'vc_item_id' => (int) $id_element,
]
);
}
$return_val = [];
$return_val['correct'] = (int) $result;
$return_val['new_line'] = $new_line;
@ -1404,9 +1430,22 @@ switch ($action) {
$values['show_on_top'] = $show_on_top;
$values['image'] = $background_color;
$values['type_graph'] = $type_graph;
$values['id_custom_graph'] = $id_custom_graph;
switch ($type) {
case 'line_item':
case 'box_item':
case 'clock':
case 'icon':
case 'label':
$values['cache_expiration'] = 0;
break;
default:
$values['cache_expiration'] = $cache_expiration;
break;
}
switch ($type) {
case 'line_item':
$values['type'] = LINE_ITEM;

View File

@ -984,6 +984,10 @@ function config_update_config()
$error_update[] = __('Use the legacy Visual Console');
}
if (!config_update_value('vc_default_cache_expiration', (int) get_parameter('vc_default_cache_expiration'))) {
$error_update[] = __("Default expiration of the Visual Console item's cache");
}
if (!config_update_value('vc_refr', (int) get_parameter('vc_refr'))) {
$error_update[] = __('Default interval for refresh on Visual Console');
}
@ -2427,6 +2431,10 @@ function config_process_config()
config_update_value('legacy_vc', 1);
}
if (!isset($config['vc_default_cache_expiration'])) {
config_update_value('vc_default_cache_expiration', 60);
}
if (!isset($config['vc_refr'])) {
config_update_value('vc_refr', 300);
}

View File

@ -1128,9 +1128,58 @@ function visual_map_editor_print_item_palette($visualConsole_id, $background)
true
).'</td>';
// Insert and modify before the buttons to create or update.
if (!$config['legacy_vc']) {
$intervals = [
10 => '10 '.__('seconds'),
30 => '30 '.__('seconds'),
60 => '1 '.__('minutes'),
300 => '5 '.__('minutes'),
900 => '15 '.__('minutes'),
1800 => '30 '.__('minutes'),
3600 => '1 '.__('hour'),
];
$form_items_advance['cache_expiration_row'] = [];
$form_items_advance['cache_expiration_row']['items'] = [
'static_graph',
'percentile_bar',
'percentile_item',
'module_graph',
'simple_value',
'datos',
'auto_sla_graph',
'group_item',
'bars_graph',
'donut_graph',
'color_cloud',
'service',
];
$form_items_advance['cache_expiration_row']['html'] = '<td align="left">';
$form_items_advance['cache_expiration_row']['html'] .= __('Cache expiration');
$form_items_advance['cache_expiration_row']['html'] .= '</td>';
$form_items_advance['cache_expiration_row']['html'] .= '<td align="left">';
$form_items_advance['cache_expiration_row']['html'] .= html_print_extended_select_for_time(
'cache_expiration',
$config['vc_default_cache_expiration'],
'',
__('No cache'),
0,
false,
true,
false,
true,
'',
false,
$intervals
);
$form_items_advance['cache_expiration_row']['html'] .= '</td>';
}
// Insert and modify before the buttons to create or update.
if (enterprise_installed()) {
enterprise_visual_map_editor_modify_form_items_advance_palette($form_items_advance);
enterprise_visual_map_editor_modify_form_items_advance_palette(
$form_items_advance
);
}
foreach ($form_items_advance as $item => $item_options) {

View File

@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
namespace Models;
use Models\Model;
/**
* This class should be extended to add functionalities to
* fetch, clear and save item cache.
*/
abstract class CachedModel extends Model
{
/**
* Used to decide if the cache should also be indexed by user or not.
*
* @var boolean
*/
protected static $indexCacheByUser = false;
/**
* Obtain a data structure from the database using a filter.
*
* @param array $filter Filter to retrieve the modeled element.
*
* @return array The modeled element data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @abstract
*/
abstract protected static function fetchCachedData(array $filter);
/**
* Stores the data structure obtained.
*
* @param array $filter Filter to retrieve the modeled element.
* @param array $data Data to store in cache.
*
* @return array The modeled element data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @abstract
*/
abstract protected static function saveCachedData(
array $filter,
array $data
): bool;
/**
* Deletes previous data that are not useful.
*
* @param array $filter Filter to retrieve the modeled element.
*
* @return array The modeled element data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @abstract
*/
abstract protected static function clearCachedData(array $filter): int;
/**
* Obtain a model's instance from the database using a filter.
*
* @param array $filter Filter to retrieve the modeled element.
*
* @return self A modeled element's instance.
*
* @overrides Model::fromDB.
*/
public static function fromDB(array $filter): Model
{
global $config;
if ($filter['cache_expiration'] > 0) {
// Obtain the item's data from cache.
$cachedData = static::fetchCachedData($filter);
if ($cachedData === null) {
$userId = (static::$indexCacheByUser === true) ? $config['id_user'] : null;
// Delete expired data cache.
static::clearCachedData(
[
'vc_item_id' => $filter['id'],
'vc_id' => $filter['id_layout'],
'user_id' => $userId,
]
);
// Obtain the item's data from the database.
$data = static::fetchDataFromDB($filter);
// Save the item's data in cache.
static::saveCachedData(
[
'vc_item_id' => $filter['id'],
'vc_id' => $filter['id_layout'],
'user_id' => $userId,
'expiration' => $filter['cache_expiration'],
],
$data
);
} else {
$data = $cachedData;
}
} else {
$data = static::fetchDataFromDB($filter);
}
return static::fromArray($data);
}
}

View File

@ -321,6 +321,8 @@ final class Container extends Model
$fields = [
'id',
'type',
'cache_expiration',
'id_layout',
];
// Override the filter if the groups filter is not empty.
@ -359,11 +361,10 @@ final class Container extends Model
foreach ($rows as $data) {
$itemId = (int) $data['id'];
$itemType = (int) $data['type'];
$class = static::getItemClass($itemType);
$class = static::getItemClass((int) $data['type']);
try {
array_push($items, $class::fromDB(['id' => $itemId]));
array_push($items, $class::fromDB($data));
} catch (\Throwable $e) {
// TODO: Log this?
}

View File

@ -3,12 +3,12 @@
declare(strict_types=1);
namespace Models\VisualConsole;
use Models\Model;
use Models\CachedModel;
/**
* Model of a generic Visual Console Item.
*/
class Item extends Model
class Item extends CachedModel
{
/**
@ -698,10 +698,6 @@ class Item extends Model
throw new \Exception('error fetching the data from the DB');
}
// Load side libraries.
global $config;
include_once $config['homedir'].'/include/functions_io.php';
// Clean up to two levels of HTML entities.
$row = \io_safe_output(\io_safe_output($row));
@ -725,6 +721,89 @@ class Item extends Model
}
/**
* Fetch a cache item data structure from the database using a filter.
*
* @param array $filter Filter of the Visual Console Item.
*
* @return array The Visual Console Item data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @override CachedModel::fetchCachedData.
*/
protected static function fetchCachedData(array $filter)
{
global $config;
$filter = [
'vc_id' => (int) $filter['id_layout'],
'vc_item_id' => (int) $filter['id'],
'(UNIX_TIMESTAMP(`created_at`) + `expiration`) > UNIX_TIMESTAMP()'
];
if (static::$indexCacheByUser === true) {
$filter['user_id'] = $config['id_user'];
}
$data = \db_get_value_filter(
'data',
'tvisual_console_elements_cache',
$filter
);
if ($data === false) {
return null;
}
return json_decode(base64_decode($data), true);
}
/**
* Stores the data structure obtained.
*
* @param array $filter Filter to save the modeled element.
* @param array $data Modeled element to save.
*
* @return boolean The modeled element data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @override CachedModel::saveCachedData.
*/
protected static function saveCachedData(array $filter, array $data): bool
{
return \db_process_sql_insert(
'tvisual_console_elements_cache',
[
'vc_id' => $filter['vc_id'],
'vc_item_id' => $filter['vc_item_id'],
'user_id' => $filter['user_id'],
'data' => base64_encode(json_encode($data)),
'expiration' => $filter['expiration'],
]
) > 0;
}
/**
* Deletes previous data that are not useful.
*
* @param array $filter Filter to retrieve the modeled element.
*
* @return array The modeled element data structure stored into the DB.
* @throws \Exception When the data cannot be retrieved from the DB.
*
* @override CachedModel::clearCachedData.
*/
protected static function clearCachedData(array $filter): int
{
return \db_process_sql_delete(
'tvisual_console_elements_cache',
$filter
);
}
/**
* Fetch a data structure of an agent from the database using the
* vs item's data.

View File

@ -26,6 +26,13 @@ final class Group extends Item
*/
protected static $useHtmlOutput = true;
/**
* Enable the cache index by user id.
*
* @var boolean
*/
protected static $indexCacheByUser = true;
/**
* Returns a valid representation of the model.

View File

@ -124,7 +124,7 @@ if ($aclWrite || $aclManage) {
).'</a>';
}
$options['view']['text'] = '<a href="index.php?sec=network&sec2=operation/visual_console/render_view&id='.$visualConsoleId.'">'.html_print_image(
$options['view']['text'] = '<a href="index.php?sec=network&sec2=operation/visual_console/render_view&id='.$visualConsoleId.'&refr='.$refr.'">'.html_print_image(
'images/operation.png',
true,
['title' => __('View')]
@ -166,7 +166,7 @@ if ($pure === true) {
// Quit fullscreen.
echo '<li class="nomn">';
$urlNoFull = 'index.php?sec=network&sec2=operation/visual_console/render_view&id='.$visualConsoleId;
$urlNoFull = 'index.php?sec=network&sec2=operation/visual_console/render_view&id='.$visualConsoleId.'&refr='.$refr;
echo '<a class="vc-btn-no-fullscreen" href="'.$urlNoFull.'">';
echo html_print_image('images/normal_screen.png', true, ['title' => __('Back to normal mode')]);
echo '</a>';

View File

@ -1537,7 +1537,7 @@ CREATE TABLE IF NOT EXISTS `tlayout_data` (
`time_format` varchar(60) NOT NULL default "time",
`timezone` varchar(60) NOT NULL default "Europe/Madrid",
`show_last_value` tinyint(1) UNSIGNED NULL default '0',
`cache_expiration` INTEGER UNSIGNED NOT NULL default 0,
PRIMARY KEY(`id`)
) ENGINE = InnoDB DEFAULT CHARSET=utf8;
@ -3569,3 +3569,24 @@ CREATE TABLE `tuser_task_scheduled` (
`id_grupo` int(10) unsigned NOT NULL default 0,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- ---------------------------------------------------------------------
-- Table `tvisual_console_items_cache`
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS `tvisual_console_elements_cache` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`vc_id` INTEGER UNSIGNED NOT NULL,
`vc_item_id` INTEGER UNSIGNED NOT NULL,
`user_id` VARCHAR(60) DEFAULT NULL,
`data` TEXT NOT NULL,
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`expiration` INTEGER UNSIGNED NOT NULL COMMENT 'Seconds to expire',
PRIMARY KEY(`id`),
FOREIGN KEY(`vc_id`) REFERENCES `tlayout`(`id`)
ON DELETE CASCADE,
FOREIGN KEY(`vc_item_id`) REFERENCES `tlayout_data`(`id`)
ON DELETE CASCADE,
FOREIGN KEY (`user_id`) REFERENCES `tusuario`(`id_user`)
ON DELETE CASCADE
ON UPDATE CASCADE
) engine=InnoDB DEFAULT CHARSET=utf8;

View File

@ -446,6 +446,10 @@ sub pandora_purgedb ($$) {
my $message_limit = time() - 86400 * $conf->{'_delete_old_messages'};
db_do ($dbh, "DELETE FROM tmensajes WHERE timestamp < ?", $message_limit);
}
# Delete old cache data
log_message ('PURGE', "Deleting old cache data.");
db_do ($dbh, "DELETE FROM `tvisual_console_elements_cache` WHERE (UNIX_TIMESTAMP(`created_at`) + `expiration`) < UNIX_TIMESTAMP()");
}
###############################################################################