From c2a44e073fe0c63c17a2f2a79288fedffb8b4c74 Mon Sep 17 00:00:00 2001 From: Daniel Maya Date: Thu, 7 Feb 2019 18:03:28 +0100 Subject: [PATCH 1/3] Update events.build_query.php Former-commit-id: 4622b27b1ce9e81ce81d89d7bc9d2eeb1b30fb93 --- .../operation/events/events.build_query.php | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/pandora_console/operation/events/events.build_query.php b/pandora_console/operation/events/events.build_query.php index 4aa9e4a567..3af11f86b7 100755 --- a/pandora_console/operation/events/events.build_query.php +++ b/pandora_console/operation/events/events.build_query.php @@ -41,6 +41,39 @@ if ($id_group > 0) { $childrens_ids = array_keys($groups); } +if (($date_from == '') && ($date_to == '')) { + if ($event_view_hr > 0) { + $filter_resume['hours_max'] = $event_view_hr; + $unixtime = (get_system_time() - ($event_view_hr * SECONDS_1HOUR)); + $sql_post .= ' AND (utimestamp > '.$unixtime.')'; + } +} else { + // Some of this values will have the user's timezone, + // so we need to reverse it to the system's timezone + // before using it into the db. + $fixed_offset = get_fixed_offset(); + + if (!empty($date_from)) { + if (empty($time_from)) { + $time_from = '00:00:00'; + } + + $utimestamp_from = (strtotime($date_from.' '.$time_from) - $fixed_offset); + $filter_resume['time_from'] = date(DATE_FORMAT.' '.TIME_FORMAT, $utimestamp_from); + $sql_post .= ' AND (utimestamp >= '.$utimestamp_from.')'; + } + + if (!empty($date_to)) { + if (empty($time_to)) { + $time_to = '23:59:59'; + } + + $utimestamp_to = (strtotime($date_to.' '.$time_to) - $fixed_offset); + $filter_resume['time_to'] = date(DATE_FORMAT.' '.TIME_FORMAT, $utimestamp_to); + $sql_post .= ' AND (utimestamp <= '.$utimestamp_to.')'; + } +} + // Group selection if ($id_group > 0 && in_array($id_group, array_keys($groups))) { if ($propagate) { @@ -51,11 +84,15 @@ if ($id_group > 0 && in_array($id_group, array_keys($groups))) { $sql_post = " AND (id_grupo = $id_group OR id_group = $id_group)"; } } else { - $sql_post = sprintf( - ' AND (id_grupo IN (%s) OR id_group IN (%s)) ', - implode(',', array_keys($groups)), - implode(',', array_keys($groups)) - ); + hd(users_can_manage_group_all('ER')); + if (!users_is_admin() && !users_can_manage_group_all('ER')) { + hd(users_can_manage_group_all('ER')); + $sql_post = sprintf( + ' AND (id_grupo IN (%s) OR id_group IN (%s)) ', + implode(',', array_keys($groups)), + implode(',', array_keys($groups)) + ); + } } // Skip system messages if user is not PM @@ -218,39 +255,6 @@ if (!isset($date_to)) { $date_to = ''; } -if (($date_from == '') && ($date_to == '')) { - if ($event_view_hr > 0) { - $filter_resume['hours_max'] = $event_view_hr; - $unixtime = (get_system_time() - ($event_view_hr * SECONDS_1HOUR)); - $sql_post .= ' AND (utimestamp > '.$unixtime.')'; - } -} else { - // Some of this values will have the user's timezone, - // so we need to reverse it to the system's timezone - // before using it into the db - $fixed_offset = get_fixed_offset(); - - if (!empty($date_from)) { - if (empty($time_from)) { - $time_from = '00:00:00'; - } - - $utimestamp_from = (strtotime($date_from.' '.$time_from) - $fixed_offset); - $filter_resume['time_from'] = date(DATE_FORMAT.' '.TIME_FORMAT, $utimestamp_from); - $sql_post .= ' AND (utimestamp >= '.$utimestamp_from.')'; - } - - if (!empty($date_to)) { - if (empty($time_to)) { - $time_to = '23:59:59'; - } - - $utimestamp_to = (strtotime($date_to.' '.$time_to) - $fixed_offset); - $filter_resume['time_to'] = date(DATE_FORMAT.' '.TIME_FORMAT, $utimestamp_to); - $sql_post .= ' AND (utimestamp <= '.$utimestamp_to.')'; - } -} - // Search by tag if (!empty($tag_with)) { if (!users_is_admin()) { @@ -398,3 +402,5 @@ if ($meta) { } } } + +hd($sql_post); From ba7540c9ee79344b81dd820d95458f24b4003aac Mon Sep 17 00:00:00 2001 From: Daniel Maya Date: Fri, 8 Feb 2019 09:51:08 +0100 Subject: [PATCH 2/3] improvement of optimization for list event Former-commit-id: a7788b01160c661139e6a3a6aac70c69441265a1 --- .../operation/events/events.build_query.php | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/pandora_console/operation/events/events.build_query.php b/pandora_console/operation/events/events.build_query.php index 3af11f86b7..85d0de9a7d 100755 --- a/pandora_console/operation/events/events.build_query.php +++ b/pandora_console/operation/events/events.build_query.php @@ -19,7 +19,6 @@ if (check_acl($id_user, 0, 'ER')) { $groups = users_get_groups($id_user, 'EM'); } - $propagate = db_get_value('propagate', 'tgrupo', 'id_grupo', $id_group); if ($id_group > 0) { @@ -41,7 +40,15 @@ if ($id_group > 0) { $childrens_ids = array_keys($groups); } -if (($date_from == '') && ($date_to == '')) { +if (!isset($date_from)) { + $date_from = ''; +} + +if (!isset($date_to)) { + $date_to = ''; +} + +if (($date_from === '') && ($date_to === '')) { if ($event_view_hr > 0) { $filter_resume['hours_max'] = $event_view_hr; $unixtime = (get_system_time() - ($event_view_hr * SECONDS_1HOUR)); @@ -74,20 +81,18 @@ if (($date_from == '') && ($date_to == '')) { } } -// Group selection +// Group selection. if ($id_group > 0 && in_array($id_group, array_keys($groups))) { if ($propagate) { $childrens_str = implode(',', $childrens_ids); - $sql_post = " AND (id_grupo IN ($childrens_str) OR id_group IN ($childrens_str))"; + $sql_post .= " AND (id_grupo IN ($childrens_str) OR id_group IN ($childrens_str))"; } else { - // If a group is selected and it's in the groups allowed - $sql_post = " AND (id_grupo = $id_group OR id_group = $id_group)"; + // If a group is selected and it's in the groups allowed. + $sql_post .= " AND (id_grupo = $id_group OR id_group = $id_group)"; } } else { - hd(users_can_manage_group_all('ER')); if (!users_is_admin() && !users_can_manage_group_all('ER')) { - hd(users_can_manage_group_all('ER')); - $sql_post = sprintf( + $sql_post .= sprintf( ' AND (id_grupo IN (%s) OR id_group IN (%s)) ', implode(',', array_keys($groups)), implode(',', array_keys($groups)) @@ -95,7 +100,7 @@ if ($id_group > 0 && in_array($id_group, array_keys($groups))) { } } -// Skip system messages if user is not PM +// Skip system messages if user is not PM. if (!check_acl($id_user, 0, 'PM')) { $sql_post .= ' AND id_grupo != 0'; } @@ -201,7 +206,7 @@ if ($source != '') { $sql_post .= " AND source LIKE '%$source%'"; } -// In metaconsole mode the agent search is performed by name +// In metaconsole mode the agent search is performed by name. if ($meta) { $text_agent = get_parameter('text_agent', ''); $id_agent = get_parameter('id_agent', 0); @@ -216,7 +221,7 @@ if ($meta) { break; case -1: - // Agent doesnt exist. No results will returned + // Agent doesnt exist. No results will returned. $sql_post .= ' AND 1 = 0'; break; @@ -229,9 +234,7 @@ if ($meta) { -if ($meta) { - // There is another filter. -} else { +if (!$meta) { if (!empty($text_module)) { $filter_resume['module'] = $text_module; $sql_post .= " AND id_agentmodule IN ( @@ -247,15 +250,7 @@ if ($id_user_ack != '0') { $sql_post .= " AND id_usuario = '".$id_user_ack."'"; } -if (!isset($date_from)) { - $date_from = ''; -} - -if (!isset($date_to)) { - $date_to = ''; -} - -// Search by tag +// Search by tag. if (!empty($tag_with)) { if (!users_is_admin()) { $user_tags = array_flip(tags_get_tags_for_module_search()); @@ -309,7 +304,7 @@ if (!empty($tag_without)) { $sql_post .= ' ) '; } -// Filter/Only alerts +// Filter/Only alerts. if (isset($filter_only_alert)) { if ($filter_only_alert == 0) { $filter_resume['alerts'] = $filter_only_alert; @@ -320,7 +315,7 @@ if (isset($filter_only_alert)) { } } -// Tags ACLS +// Tags ACLS. if ($id_group > 0 && in_array($id_group, array_keys($groups))) { $group_array = (array) $id_group; } else { @@ -339,7 +334,7 @@ if (check_acl($id_user, 0, 'ER')) { [], true ); - // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)" + // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)". } else if (check_acl($id_user, 0, 'EW')) { $tags_acls_condition = tags_get_acl_tags( $id_user, @@ -352,7 +347,7 @@ if (check_acl($id_user, 0, 'ER')) { [], true ); - // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)" + // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)". } else if (check_acl($id_user, 0, 'EM')) { $tags_acls_condition = tags_get_acl_tags( $id_user, @@ -365,14 +360,14 @@ if (check_acl($id_user, 0, 'ER')) { [], true ); - // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)" + // FORCE CHECK SQL "(TAG = tag1 AND id_grupo = 1)". } if (($tags_acls_condition != ERR_WRONG_PARAMETERS) && ($tags_acls_condition != ERR_ACL) && ($tags_acls_condition != -110000)) { $sql_post .= $tags_acls_condition; } -// Metaconsole fitlers +// Metaconsole fitlers. if ($meta) { if ($server_id) { $filter_resume['server'] = $server_id; @@ -402,5 +397,3 @@ if ($meta) { } } } - -hd($sql_post); From 4c809ba8d2e68755b7ed11b2cf5557446324eb77 Mon Sep 17 00:00:00 2001 From: Daniel Maya Date: Wed, 13 Feb 2019 15:41:34 +0100 Subject: [PATCH 3/3] event query optimization Former-commit-id: 322868d281ddeac4006480bac6dbf03434ffefa1 --- pandora_console/include/functions_events.php | 124 +++++-------------- 1 file changed, 28 insertions(+), 96 deletions(-) diff --git a/pandora_console/include/functions_events.php b/pandora_console/include/functions_events.php index f3f3d2b0fa..4cc54c5857 100644 --- a/pandora_console/include/functions_events.php +++ b/pandora_console/include/functions_events.php @@ -166,92 +166,30 @@ function events_get_events_grouped( $groupby_extra = ''; } - switch ($config['dbtype']) { - case 'mysql': - db_process_sql('SET group_concat_max_len = 9999999'); - $event_lj = events_get_secondary_groups_left_join($table); - if ($total) { - $sql = "SELECT COUNT(*) FROM (SELECT * - FROM $table te $event_lj - WHERE 1=1 ".$sql_post.' - GROUP BY estado, evento, id_agente, id_agentmodule'.$groupby_extra.') AS t'; - } else { - $sql = "SELECT *, MAX(id_evento) AS id_evento, - GROUP_CONCAT(DISTINCT user_comment SEPARATOR '
') AS user_comment, - GROUP_CONCAT(DISTINCT id_evento SEPARATOR ',') AS similar_ids, - COUNT(*) AS event_rep, MAX(utimestamp) AS timestamp_rep, - MIN(utimestamp) AS timestamp_rep_min, - (SELECT owner_user FROM $table WHERE id_evento = MAX(te.id_evento)) owner_user, - (SELECT id_usuario FROM $table WHERE id_evento = MAX(te.id_evento)) id_usuario, - (SELECT id_agente FROM $table WHERE id_evento = MAX(te.id_evento)) id_agente, - (SELECT criticity FROM $table WHERE id_evento = MAX(te.id_evento)) AS criticity, - (SELECT ack_utimestamp FROM $table WHERE id_evento = MAX(te.id_evento)) AS ack_utimestamp, - (SELECT nombre FROM tagente_modulo WHERE id_agente_modulo = te.id_agentmodule) AS module_name - FROM $table te $event_lj - WHERE 1=1 ".$sql_post.' - GROUP BY estado, evento, id_agente, id_agentmodule'.$groupby_extra; - $sql .= ' '.events_get_sql_order($sort_field, $order, 2); - $sql .= ' LIMIT '.$offset.','.$pagination; - } - break; - - case 'postgresql': - if ($total) { - $sql = "SELECT COUNT(*) - FROM $table te - WHERE 1=1 ".$sql_post.' - GROUP BY estado, evento, id_agentmodule, id_evento, id_agente, id_usuario, id_grupo, estado, timestamp, utimestamp, event_type, id_alert_am, criticity, user_comment, tags, source, id_extra'.$groupby_extra; - } else { - $sql = "SELECT *, MAX(id_evento) AS id_evento, array_to_string(array_agg(DISTINCT user_comment), '
') AS user_comment, - array_to_string(array_agg(DISTINCT id_evento), ',') AS similar_ids, - COUNT(*) AS event_rep, MAX(utimestamp) AS timestamp_rep, - MIN(utimestamp) AS timestamp_rep_min, - (SELECT owner_user FROM $table WHERE id_evento = MAX(te.id_evento)) owner_user, - (SELECT id_usuario FROM $table WHERE id_evento = MAX(te.id_evento)) id_usuario, - (SELECT id_agente FROM $table WHERE id_evento = MAX(te.id_evento)) id_agente, - (SELECT criticity FROM $table WHERE id_evento = MAX(te.id_evento)) AS criticity, - (SELECT ack_utimestamp FROM $table WHERE id_evento = MAX(te.id_evento)) AS ack_utimestamp - FROM $table te - WHERE 1=1 ".$sql_post.' - GROUP BY estado, evento, id_agentmodule, id_evento, - id_agente, id_usuario, id_grupo, estado, - timestamp, utimestamp, event_type, id_alert_am, - criticity, user_comment, tags, source, id_extra, - te.critical_instructions, - te.warning_instructions, - te.unknown_instructions, - te.owner_user, - te.ack_utimestamp, - te.custom_data '.$groupby_extra.' - ORDER BY timestamp_rep ASC LIMIT '.$pagination.' OFFSET '.$offset; - } - break; - - case 'oracle': - if ($total) { - $sql = "SELECT COUNT(*) - FROM $table te - WHERE 1=1 $sql_post - GROUP BY estado, to_char(evento), id_agentmodule".$groupby_extra.') b '; - } else { - $set = []; - $set['limit'] = $pagination; - $set['offset'] = $offset; - - $sql = "SELECT ta.*, tb.event_rep, tb.timestamp_rep, tb.timestamp_rep_min, tb.user_comments, tb.similar_ids - FROM $table ta - INNER JOIN (SELECT MAX(id_evento) AS id_evento, COUNT(id_evento) AS event_rep, - MAX(utimestamp) AS timestamp_rep, MIN(utimestamp) AS timestamp_rep_min, - TAB_TO_STRING(CAST(COLLECT(TO_CHAR(user_comment) ORDER BY id_evento ASC) AS t_varchar2_tab), '
') AS user_comments, - TAB_TO_STRING(CAST(COLLECT(CAST(id_evento AS VARCHAR2(4000)) ORDER BY id_evento ASC) AS t_varchar2_tab)) AS similar_ids - FROM $table te - WHERE 1=1 $sql_post - GROUP BY estado, to_char(evento), id_agentmodule$groupby_extra) tb - ON ta.id_evento = tb.id_evento - ORDER BY tb.timestamp_rep ASC"; - $sql = oracle_recode_query($sql, $set); - } - break; + db_process_sql('SET group_concat_max_len = 9999999'); + $event_lj = events_get_secondary_groups_left_join($table); + if ($total) { + $sql = "SELECT COUNT(*) FROM (SELECT id_evento + FROM $table te $event_lj + WHERE 1=1 ".$sql_post.' + GROUP BY estado, evento, id_agente, id_agentmodule'.$groupby_extra.') AS t'; + } else { + $sql = "SELECT *, MAX(id_evento) AS id_evento, + GROUP_CONCAT(DISTINCT user_comment SEPARATOR '
') AS user_comment, + GROUP_CONCAT(DISTINCT id_evento SEPARATOR ',') AS similar_ids, + COUNT(id_evento) AS event_rep, MAX(utimestamp) AS timestamp_rep, + MIN(utimestamp) AS timestamp_rep_min, + (SELECT owner_user FROM $table WHERE id_evento = MAX(te.id_evento)) owner_user, + (SELECT id_usuario FROM $table WHERE id_evento = MAX(te.id_evento)) id_usuario, + (SELECT id_agente FROM $table WHERE id_evento = MAX(te.id_evento)) id_agente, + (SELECT criticity FROM $table WHERE id_evento = MAX(te.id_evento)) AS criticity, + (SELECT ack_utimestamp FROM $table WHERE id_evento = MAX(te.id_evento)) AS ack_utimestamp, + (SELECT nombre FROM tagente_modulo WHERE id_agente_modulo = te.id_agentmodule) AS module_name + FROM $table te $event_lj + WHERE 1=1 ".$sql_post.' + GROUP BY estado, evento, id_agente, id_agentmodule'.$groupby_extra; + $sql .= ' '.events_get_sql_order($sort_field, $order, 2); + $sql .= ' LIMIT '.$offset.','.$pagination; } // Extract the events by filter (or not) from db @@ -260,16 +198,6 @@ function events_get_events_grouped( if ($total) { return reset($events[0]); } else { - // Override the column 'user_comment' with the column 'user_comments' when oracle - if (!empty($events) && $config['dbtype'] == 'oracle') { - array_walk( - $events, - function (&$value, $key) { - set_if_defined($value['user_comment'], $value['user_comments']); - } - ); - } - return $events; } } @@ -4871,6 +4799,10 @@ function events_get_sql_order($sort_field='timestamp', $sort='DESC', $group_rep= */ function events_get_secondary_groups_left_join($table) { + if (users_is_admin()) { + return ''; + } + if ($table == 'tevento') { return 'LEFT JOIN tagent_secondary_group tasg ON te.id_agente = tasg.id_agent'; }