From 40ea25eeea94a678434b90d4ec34692382913f53 Mon Sep 17 00:00:00 2001 From: Calvo Date: Thu, 22 Jun 2023 18:52:06 +0200 Subject: [PATCH 1/4] Clean deleted agents from dashboards, reports, etc --- pandora_console/include/functions_agents.php | 63 ++++++++++++++++++++ pandora_server/util/pandora_db.pl | 50 +++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/pandora_console/include/functions_agents.php b/pandora_console/include/functions_agents.php index f8bb2fc024..2c5ca86e7f 100644 --- a/pandora_console/include/functions_agents.php +++ b/pandora_console/include/functions_agents.php @@ -2769,6 +2769,69 @@ function agents_delete_agent($id_agents, $disableACL=false) enterprise_include_once('include/functions_agents.php'); enterprise_hook('agent_delete_from_cache', [$id_agent]); + // Delete agent from visual console. + db_process_sql_delete( + 'tlayout_data', + ['id_agent' => $id_agent] + ); + + // Delete agent from visual dashboards. + db_process_sql( + 'UPDATE twidget_dashboard + SET options = NULL + WHERE options LIKE ("%\"agentid\":\"'.$id_agent.'\"%")' + ); + + // Delete agent from treport. + db_process_sql_delete( + 'treport_content', + ['id_agent' => $id_agent] + ); + + // Delete rules from tevent alerts (correlative alerts) + db_process_sql_delete( + 'tevent_rule', + [ + 'agent' => $id_agent, + 'operator_agent' => '==', + ] + ); + + db_process_sql_delete( + 'tevent_rule', + [ + 'log_agent' => $id_agent, + 'operator_log_agent' => '==', + ] + ); + + // Delete from gis maps history + db_process_sql_delete( + 'tgis_data_history', + ['tagente_id_agente' => $id_agent] + ); + + // Delete from policies. + db_process_sql_delete( + 'tpolicy_agents', + ['id_agent' => $id_agent] + ); + + // Delete from tnetwork maps + db_process_sql_delete( + 'titem', + ['source_data' => $id_agent] + ); + + db_process_sql_delete( + 'trel_item', + [ + 'id_parent_source_data' => $id_agent, + 'id_child_source_data' => $id_agent, + ], + 'OR' + ); + // Delete agent from fav menu. db_process_sql_delete( 'tfavmenu_user', diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index 31218a8f19..48ea46f7fe 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -760,7 +760,55 @@ sub pandora_checkdb_integrity { # Delete orphan data_inc reference records db_do ($dbh, 'DELETE FROM tagente_datos_inc WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); - + + # Delete orphan data form deleted agents. + my @agents_ids = get_db_rows ($dbh, 'SELECT id_agente, alias FROM tagente'); + my $agents_id = '0'; + my $agents_alias; + foreach my $id (@agents_ids) { + $agents_id .= ','.$id->{'id_agente'}; + $agents_alias .= ','.$id->{'alias'}; + } + if(defined($agents_id) && $agents_id ne '0') { + # Delete orphan data from visual console. + db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agent NOT IN (?)', $agents_id); + + # Clearl orphan data from dashboards + my $where_condition; + my $index ; + foreach my $agent_id (@agents_ids) { + $where_condition .= 'options NOT LIKE ("%\\"agentid\\":\\"'.$agent_id->{'id_agente'}.'\\"%")'; + if($agent_id == @agents_ids[-1]) { + last; + } + $where_condition .= ' AND '; + } + db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); + + # Delete orphan report items. + db_do ($dbh, 'DELETE FROM treport_content WHERE id_agent != 0 AND id_agent NOT IN (?)', $agents_id); + + # Delete orphan tevent alert rules + db_do ($dbh, 'DELETE FROM tevent_rule WHERE agent IS NOT NULL AND agent != "" AND agent NOT IN (?) AND operator_agent = ?', $agents_alias, '=='); + db_do ($dbh, 'DELETE FROM tevent_rule WHERE log_agent IS NOT NULL AND log_agent != "" AND log_agent NOT IN (?) AND operator_log_agent = ?', $agents_alias, '=='); + + # Delete orphan data from favorite agents + db_do ($dbh, 'DELETE FROM tfavmenu_user WHERE section = "Agents" AND id_element NOT IN (?)', $agents_id); + + # Delete orphan data from tservices. + db_do ($dbh, 'DELETE FROM tservice_element WHERE id_agent NOT IN (?)', $agents_id); + + # Delete orphan data from gis maps + db_do ($dbh, 'DELETE FROM tgis_data_history WHERE tagente_id_agente NOT IN (?)', $agents_id); + + # Delete agents from policies + db_do ($dbh, 'DELETE FROM tpolicy_agents WHERE id_agent NOT IN (?)', $agents_id); + + # Delete orphan tnetwork maps data + db_do ($dbh, 'DELETE FROM titem WHERE source_data NOT IN (?)', $agents_id); + db_do ($dbh, 'DELETE FROM trel_item WHERE id_parent_source_data NOT IN (?) OR id_child_source_data NOT IN (?)', $agents_id, $agents_id); + } + # Check enterprise tables enterprise_hook ('pandora_checkdb_integrity_enterprise', [$conf, $dbh]); } From 7a3da419bd0efd2c97103bd45365cdf1c26953a2 Mon Sep 17 00:00:00 2001 From: Calvo Date: Fri, 23 Jun 2023 15:06:31 +0200 Subject: [PATCH 2/4] Clean deleted agents and modules from dashboards, reports, etc --- pandora_server/util/pandora_db.pl | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index 48ea46f7fe..d8d61eb7df 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -764,51 +764,55 @@ sub pandora_checkdb_integrity { # Delete orphan data form deleted agents. my @agents_ids = get_db_rows ($dbh, 'SELECT id_agente, alias FROM tagente'); my $agents_id = '0'; - my $agents_alias; foreach my $id (@agents_ids) { $agents_id .= ','.$id->{'id_agente'}; - $agents_alias .= ','.$id->{'alias'}; } if(defined($agents_id) && $agents_id ne '0') { # Delete orphan data from visual console. + log_message ('INTEGRITY', "Deleting orphan visual console items."); db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agent NOT IN (?)', $agents_id); + db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); # Clearl orphan data from dashboards + log_message ('INTEGRITY', "Deleting orphan dahsboard items."); my $where_condition; - my $index ; foreach my $agent_id (@agents_ids) { $where_condition .= 'options NOT LIKE ("%\\"agentid\\":\\"'.$agent_id->{'id_agente'}.'\\"%")'; - if($agent_id == @agents_ids[-1]) { + if($agent_id == $agents_ids[-1]) { last; } $where_condition .= ' AND '; } + db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); - # Delete orphan report items. - db_do ($dbh, 'DELETE FROM treport_content WHERE id_agent != 0 AND id_agent NOT IN (?)', $agents_id); + $where_condition = ''; + my @modules = get_db_rows($dbh, 'SELECT id_agente_modulo FROM tagente_modulo'); + foreach my $id_agente_modulo (@modules) { + print Dumper($id_agente_modulo); + $where_condition .= 'options NOT LIKE ("%\\"moduleId\\":\\"'.$id_agente_modulo->{'id_agente_modulo'}.'\\"%")'; + if($id_agente_modulo == $modules[-1]) { + last; + } + $where_condition .= ' AND '; + } - # Delete orphan tevent alert rules - db_do ($dbh, 'DELETE FROM tevent_rule WHERE agent IS NOT NULL AND agent != "" AND agent NOT IN (?) AND operator_agent = ?', $agents_alias, '=='); - db_do ($dbh, 'DELETE FROM tevent_rule WHERE log_agent IS NOT NULL AND log_agent != "" AND log_agent NOT IN (?) AND operator_log_agent = ?', $agents_alias, '=='); + db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); # Delete orphan data from favorite agents + log_message ('INTEGRITY', "Deleting orphan favories items."); db_do ($dbh, 'DELETE FROM tfavmenu_user WHERE section = "Agents" AND id_element NOT IN (?)', $agents_id); - # Delete orphan data from tservices. - db_do ($dbh, 'DELETE FROM tservice_element WHERE id_agent NOT IN (?)', $agents_id); - # Delete orphan data from gis maps + log_message ('INTEGRITY', "Deleting orphan GIS data."); db_do ($dbh, 'DELETE FROM tgis_data_history WHERE tagente_id_agente NOT IN (?)', $agents_id); - # Delete agents from policies - db_do ($dbh, 'DELETE FROM tpolicy_agents WHERE id_agent NOT IN (?)', $agents_id); - # Delete orphan tnetwork maps data + log_message ('INTEGRITY', "Deleting orphan networkmaps data."); db_do ($dbh, 'DELETE FROM titem WHERE source_data NOT IN (?)', $agents_id); db_do ($dbh, 'DELETE FROM trel_item WHERE id_parent_source_data NOT IN (?) OR id_child_source_data NOT IN (?)', $agents_id, $agents_id); } - + # Check enterprise tables enterprise_hook ('pandora_checkdb_integrity_enterprise', [$conf, $dbh]); } From d3fded0fa88a1143a9e0d814e917c415f2b4ad96 Mon Sep 17 00:00:00 2001 From: Calvo Date: Mon, 17 Jul 2023 12:02:48 +0200 Subject: [PATCH 3/4] Deleted missing Dumper --- pandora_server/util/pandora_db.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index d8d61eb7df..671ac819fc 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -789,7 +789,6 @@ sub pandora_checkdb_integrity { $where_condition = ''; my @modules = get_db_rows($dbh, 'SELECT id_agente_modulo FROM tagente_modulo'); foreach my $id_agente_modulo (@modules) { - print Dumper($id_agente_modulo); $where_condition .= 'options NOT LIKE ("%\\"moduleId\\":\\"'.$id_agente_modulo->{'id_agente_modulo'}.'\\"%")'; if($id_agente_modulo == $modules[-1]) { last; From 24b555da87c1bb37abab33c12cae6f7cd138c96d Mon Sep 17 00:00:00 2001 From: Calvo Date: Mon, 17 Jul 2023 16:31:40 +0200 Subject: [PATCH 4/4] Optimize pandora_db iterations when possible --- pandora_server/util/pandora_db.pl | 85 +++++++++++++++---------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index 671ac819fc..c3648d6fc6 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -761,56 +761,51 @@ sub pandora_checkdb_integrity { # Delete orphan data_inc reference records db_do ($dbh, 'DELETE FROM tagente_datos_inc WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); + # Delete orphan data from visual console. + log_message ('INTEGRITY', "Deleting orphan visual console items."); + db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agent NOT IN (SELECT id_agente FROM tagente)'); + db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); + # Delete orphan data form deleted agents. - my @agents_ids = get_db_rows ($dbh, 'SELECT id_agente, alias FROM tagente'); - my $agents_id = '0'; - foreach my $id (@agents_ids) { - $agents_id .= ','.$id->{'id_agente'}; + # Clearl orphan data from dashboards + log_message ('INTEGRITY', "Deleting orphan dahsboard items."); + my @agents_ids = get_db_rows($dbh, 'SELECT id_agente FROM tagente'); + my $where_condition; + foreach my $agent_id (@agents_ids) { + $where_condition .= 'options NOT LIKE ("%\\"agentid\\":\\"'.$agent_id->{'id_agente'}.'\\"%")'; + if($agent_id == $agents_ids[-1]) { + last; + } + $where_condition .= ' AND '; } - if(defined($agents_id) && $agents_id ne '0') { - # Delete orphan data from visual console. - log_message ('INTEGRITY', "Deleting orphan visual console items."); - db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agent NOT IN (?)', $agents_id); - db_do ($dbh, 'DELETE FROM tlayout_data WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); - # Clearl orphan data from dashboards - log_message ('INTEGRITY', "Deleting orphan dahsboard items."); - my $where_condition; - foreach my $agent_id (@agents_ids) { - $where_condition .= 'options NOT LIKE ("%\\"agentid\\":\\"'.$agent_id->{'id_agente'}.'\\"%")'; - if($agent_id == $agents_ids[-1]) { - last; - } - $where_condition .= ' AND '; - } + db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); - db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); - - $where_condition = ''; - my @modules = get_db_rows($dbh, 'SELECT id_agente_modulo FROM tagente_modulo'); - foreach my $id_agente_modulo (@modules) { - $where_condition .= 'options NOT LIKE ("%\\"moduleId\\":\\"'.$id_agente_modulo->{'id_agente_modulo'}.'\\"%")'; - if($id_agente_modulo == $modules[-1]) { - last; - } - $where_condition .= ' AND '; - } - - db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); - - # Delete orphan data from favorite agents - log_message ('INTEGRITY', "Deleting orphan favories items."); - db_do ($dbh, 'DELETE FROM tfavmenu_user WHERE section = "Agents" AND id_element NOT IN (?)', $agents_id); - - # Delete orphan data from gis maps - log_message ('INTEGRITY', "Deleting orphan GIS data."); - db_do ($dbh, 'DELETE FROM tgis_data_history WHERE tagente_id_agente NOT IN (?)', $agents_id); - - # Delete orphan tnetwork maps data - log_message ('INTEGRITY', "Deleting orphan networkmaps data."); - db_do ($dbh, 'DELETE FROM titem WHERE source_data NOT IN (?)', $agents_id); - db_do ($dbh, 'DELETE FROM trel_item WHERE id_parent_source_data NOT IN (?) OR id_child_source_data NOT IN (?)', $agents_id, $agents_id); + $where_condition = ''; + my @modules = get_db_rows($dbh, 'SELECT id_agente_modulo FROM tagente_modulo'); + foreach my $id_agente_modulo (@modules) { + $where_condition .= 'options NOT LIKE ("%\\"moduleId\\":\\"'.$id_agente_modulo->{'id_agente_modulo'}.'\\"%")'; + if($id_agente_modulo == $modules[-1]) { + last; + } + $where_condition .= ' AND '; } + + db_do ($dbh, 'UPDATE twidget_dashboard set options = NULL WHERE '.$where_condition); + + # Delete orphan data from favorite agents + log_message ('INTEGRITY', "Deleting orphan favories items."); + db_do ($dbh, 'DELETE FROM tfavmenu_user WHERE section = "Agents" AND id_element NOT IN (SELECT id_agente FROM tagente)'); + + # Delete orphan data from gis maps + log_message ('INTEGRITY', "Deleting orphan GIS data."); + db_do ($dbh, 'DELETE FROM tgis_data_history WHERE tagente_id_agente NOT IN (SELECT id_agente FROM tagente)'); + + # Delete orphan tnetwork maps data + log_message ('INTEGRITY', "Deleting orphan networkmaps data."); + db_do ($dbh, 'DELETE FROM titem WHERE source_data NOT IN (SELECT id_agente FROM tagente)'); + db_do ($dbh, 'DELETE FROM trel_item WHERE id_parent_source_data NOT IN (SELECT id_agente FROM tagente) OR id_child_source_data NOT IN (SELECT id_agente FROM tagente)'); + # Check enterprise tables enterprise_hook ('pandora_checkdb_integrity_enterprise', [$conf, $dbh]);