From 2c477e694751d8b9078c1fd87d81cd9438b3a3a2 Mon Sep 17 00:00:00 2001 From: ramonn Date: Wed, 15 Jan 2014 11:10:53 +0000 Subject: [PATCH] 2014-01-15 Ramon Novoa * bin/pandora_server: Prevent the thread that executes periodic tasks from dying on database errors. git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@9324 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f --- pandora_server/ChangeLog | 5 + pandora_server/bin/pandora_server | 157 +++++++++++++++--------------- 2 files changed, 84 insertions(+), 78 deletions(-) diff --git a/pandora_server/ChangeLog b/pandora_server/ChangeLog index cb97086925..dd24b1a828 100644 --- a/pandora_server/ChangeLog +++ b/pandora_server/ChangeLog @@ -1,3 +1,8 @@ +2014-01-15 Ramon Novoa + + * bin/pandora_server: Prevent the thread that executes periodic tasks + from dying on database errors. + 2014-01-14 Sergio Martin * util/recon_scripts/snmpdevices.pl: Added address check diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index d434d3546b..4e36f68e8d 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -244,88 +244,89 @@ sub pandora_server_tasks ($) { my $counter = 0; while (1) { - - # TASKS EXECUTED EVERY 5 SECONDS (Low latency tasks) - # -------------------------------------------------- - if (($counter % 5) == 0) { - - # Update forced alerts - pandora_exec_forced_alerts ($pa_config, $dbh); - } - - # TASKS EXECUTED EVERY 30 SECONDS (Mid latency tasks) - # --------------------------------------------------- - if (($counter % 30) == 0) { - - # Update module status and fired alert counts - my @agents = get_db_rows ($dbh, 'SELECT id_agente, nombre, update_module_count, update_alert_count FROM tagente WHERE disabled = 0 AND (update_module_count=1 OR update_alert_count=1)'); - foreach my $agent (@agents) { - logger ($pa_config, "Updating module status and fired alert counts for agent " . $agent->{'nombre'}, 10); - - if ($agent->{'update_module_count'} == 1) { - pandora_update_agent_module_count ($dbh, $agent->{'id_agente'}); - } - - if ($agent->{'update_alert_count'} == 1) { - pandora_update_agent_alert_count ($dbh, $agent->{'id_agente'}); - } - } - - # Keepalive module control.(very DB intensive, not run frecuently - pandora_module_keep_alive_nd ($pa_config, $dbh); - - # Set the status of unknown modules - pandora_module_unknown ($pa_config, $dbh); - - # Set event storm protection - pandora_set_event_storm_protection (pandora_get_tconfig_token ($dbh, 'event_storm_protection', 0)); - } - - # TASKS EXECUTED EVERY 60 SECONDS (High latency tasks) - # ---------------------------------------------------- - if (($counter % 60) == 0) { - # Downtimes are executed only 30 x Server Threshold secs - pandora_planned_downtime ($pa_config, $dbh); - - # Realtime stats (Only master server!) - ( VERY HEAVY !) - # Realtimestats == 1, generated by WEB Console, not by server! - if ($pa_config->{"pandora_master"} == 1 - && defined($pa_config->{"realtimestats"}) - && $pa_config->{"realtimestats"} == 0){ - - # Check if I need to refresh stats - my $last_execution_stats = get_db_value ($dbh, "SELECT MAX(utimestamp) FROM tgroup_stat"); - if (!defined($last_execution_stats) || $last_execution_stats < (time() - $pa_config->{"stats_interval"})){ - pandora_group_statistics ($pa_config, $dbh); - pandora_server_statistics ($pa_config, $dbh); - } + eval{ + # TASKS EXECUTED EVERY 5 SECONDS (Low latency tasks) + # -------------------------------------------------- + if (($counter % 5) == 0) { + + # Update forced alerts + pandora_exec_forced_alerts ($pa_config, $dbh); } - # Pandora self monitoring - if (defined($pa_config->{"self_monitoring"}) - && $pa_config->{"self_monitoring"} == 1){ - pandora_self_monitoring ($pa_config, $dbh); - } - - # Event auto-expiry - my $expiry_time = $pa_config->{"event_expiry_time"}; - my $expiry_window = $pa_config->{"event_expiry_window"}; - if ($expiry_time > 0 && $expiry_window > 0 && $expiry_window > $expiry_time) { - my $time_ref = time (); - my $expiry_limit = $time_ref - $expiry_time; - my $expiry_window = $time_ref - $expiry_window; - db_do ($dbh, 'UPDATE tevento SET estado=1, ack_utimestamp=? WHERE estado=0 AND utimestamp < ? AND utimestamp > ?', $time_ref, $expiry_limit, $expiry_window); - } - } + # TASKS EXECUTED EVERY 30 SECONDS (Mid latency tasks) + # --------------------------------------------------- + if (($counter % 30) == 0) { - # Avoid counter overflow - if ($counter > 10000){ - $counter = 0; - } - else { - $counter++; - } + # Update module status and fired alert counts + my @agents = get_db_rows ($dbh, 'SELECT id_agente, nombre, update_module_count, update_alert_count FROM tagente WHERE disabled = 0 AND (update_module_count=1 OR update_alert_count=1)'); + foreach my $agent (@agents) { + logger ($pa_config, "Updating module status and fired alert counts for agent " . $agent->{'nombre'}, 10); + + if ($agent->{'update_module_count'} == 1) { + pandora_update_agent_module_count ($dbh, $agent->{'id_agente'}); + } + + if ($agent->{'update_alert_count'} == 1) { + pandora_update_agent_alert_count ($dbh, $agent->{'id_agente'}); + } + } + + # Keepalive module control.(very DB intensive, not run frecuently + pandora_module_keep_alive_nd ($pa_config, $dbh); + + # Set the status of unknown modules + pandora_module_unknown ($pa_config, $dbh); + + # Set event storm protection + pandora_set_event_storm_protection (pandora_get_tconfig_token ($dbh, 'event_storm_protection', 0)); + } + + # TASKS EXECUTED EVERY 60 SECONDS (High latency tasks) + # ---------------------------------------------------- + if (($counter % 60) == 0) { + # Downtimes are executed only 30 x Server Threshold secs + pandora_planned_downtime ($pa_config, $dbh); + + # Realtime stats (Only master server!) - ( VERY HEAVY !) + # Realtimestats == 1, generated by WEB Console, not by server! + if ($pa_config->{"pandora_master"} == 1 + && defined($pa_config->{"realtimestats"}) + && $pa_config->{"realtimestats"} == 0){ + + # Check if I need to refresh stats + my $last_execution_stats = get_db_value ($dbh, "SELECT MAX(utimestamp) FROM tgroup_stat"); + if (!defined($last_execution_stats) || $last_execution_stats < (time() - $pa_config->{"stats_interval"})){ + pandora_group_statistics ($pa_config, $dbh); + pandora_server_statistics ($pa_config, $dbh); + } + } + + # Pandora self monitoring + if (defined($pa_config->{"self_monitoring"}) + && $pa_config->{"self_monitoring"} == 1){ + pandora_self_monitoring ($pa_config, $dbh); + } + + # Event auto-expiry + my $expiry_time = $pa_config->{"event_expiry_time"}; + my $expiry_window = $pa_config->{"event_expiry_window"}; + if ($expiry_time > 0 && $expiry_window > 0 && $expiry_window > $expiry_time) { + my $time_ref = time (); + my $expiry_limit = $time_ref - $expiry_time; + my $expiry_window = $time_ref - $expiry_window; + db_do ($dbh, 'UPDATE tevento SET estado=1, ack_utimestamp=? WHERE estado=0 AND utimestamp < ? AND utimestamp > ?', $time_ref, $expiry_limit, $expiry_window); + } + } + # Avoid counter overflow + if ($counter > 10000){ + $counter = 0; + } + else { + $counter++; + } + }; + sleep (1); } }