diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index 7e60690ae1..408472c113 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -101,8 +101,21 @@ sub pandora_startup () { # Generate the encryption key after reading the passphrase. $Config{"encryption_key"} = enterprise_hook('pandora_get_encryption_key', [\%Config, $Config{"encryption_passphrase"}]); - # Update the agent cache. - threads->create('enterprise_hook', ('update_agent_cache', [\%Config]))->detach() if ($Config{'node_metaconsole'} == 1); + # Kill any running server threads. + stop_server_threads(); + + # Start the task execution thread. + start_server_thread(\&pandora_server_tasks, [\%Config]); + + # Start the policy queue thread. + start_server_thread(\&pandora_process_policy_queue, [\%Config]) if ($Config{'__enterprise_enabled'} == 1 && $Config{'policy_manager'} == 1); + + # Start the event replication thread. Do not start with start_server_thread, this thread may exit on its own. + threads->create(\&pandora_process_event_replication, [\%Config]) if($Config{'__enterprise_enabled'} == 1 && $Config{'event_replication'} == 1); + + # Update the agent cache. Do not start with start_server_thread, this thread updates the agent cache and exits. + threads->create(\&enterprise_hook, ['update_agent_cache', [\%Config]])->detach() if ($Config{'node_metaconsole'} == 1); + pandora_audit (\%Config, $Config{'rb_product_name'} . ' Server Daemon starting', 'SYSTEM', 'System', $DBH); # Load servers @@ -143,9 +156,11 @@ sub pandora_restart (;$) { my $sleep_time = @_ > 0 ? $_[0] : $Config{'restart_delay'}; # Stop the servers - foreach my $server (@Servers) { - $server->stop (); - } + eval { + foreach my $server (@Servers) { + $server->stop (); + } + }; # Remove the servers while (pop (@Servers)) {}; @@ -274,111 +289,110 @@ sub pandora_server_tasks ($) { # Get the console DB connection my $dbh = db_connect ($pa_config->{'dbengine'}, $pa_config->{'dbname'}, $pa_config->{'dbhost'}, $pa_config->{'dbport'}, $pa_config->{'dbuser'}, $pa_config->{'dbpass'}); - my $counter = 0; - while ($RUN == 1) { - eval{ - if (pandora_is_master($pa_config) == 1) { + while ($THRRUN == 1) { + if (pandora_is_master($pa_config) == 1) { - # TASKS EXECUTED EVERY 5 SECONDS (Low latency tasks) - # -------------------------------------------------- - if (($counter % 5) == 0) { + # TASKS EXECUTED EVERY 5 SECONDS (Low latency tasks) + # -------------------------------------------------- + if (($counter % 5) == 0) { - # Update forced alerts - pandora_exec_forced_alerts ($pa_config, $dbh); - } - - # TASKS EXECUTED EVERY 30 SECONDS (Mid latency tasks) - # --------------------------------------------------- - if (($counter % 30) == 0) { - - # Update module status and fired alert counts - my @agents = get_db_rows ($dbh, 'SELECT id_agente, nombre, update_module_count, update_alert_count, update_secondary_groups FROM tagente WHERE disabled = 0 AND (update_module_count=1 OR update_alert_count=1 OR update_secondary_groups=1)'); - foreach my $agent (@agents) { - logger ($pa_config, "Updating module status and fired alert counts for agent " . $agent->{'nombre'}, 10); - - if ($agent->{'update_module_count'} == 1) { - pandora_update_agent_module_count ($pa_config, $dbh, $agent->{'id_agente'}); - } - - if ($agent->{'update_alert_count'} == 1) { - pandora_update_agent_alert_count ($pa_config, $dbh, $agent->{'id_agente'}); - } - - if ($agent->{'update_secondary_groups'} == 1) { - pandora_update_secondary_groups_cache ($pa_config, $dbh, $agent->{'id_agente'}); - } - } - - # Keepalive module control.(very DB intensive, not run frecuently - pandora_module_keep_alive_nd ($pa_config, $dbh); - - # Set the status of unknown modules - pandora_module_unknown ($pa_config, $dbh); - - # Check if an autodisabled agent needs to be autodisable - pandora_disable_autodisable_agents ($pa_config, $dbh); - } - - # TASKS EXECUTED EVERY 60 SECONDS (High latency tasks) - # ---------------------------------------------------- - if (($counter % 60) == 0) { - # Downtimes are executed only 30 x Server Threshold secs - pandora_planned_downtime ($pa_config, $dbh); - - # Realtime stats (Only master server!) - ( VERY HEAVY !) - # Realtimestats == 1, generated by WEB Console, not by server! - if (defined($pa_config->{"realtimestats"}) && $pa_config->{"realtimestats"} == 0){ - - # Check if I need to refresh stats - my $last_execution_stats = get_db_value ($dbh, "SELECT MAX(utimestamp) FROM tgroup_stat"); - if (!defined($last_execution_stats) || $last_execution_stats < (time() - $pa_config->{"stats_interval"})){ - pandora_group_statistics ($pa_config, $dbh); - pandora_server_statistics ($pa_config, $dbh); - } - } - - # Event auto-expiry - my $expiry_time = $pa_config->{"event_expiry_time"}; - my $expiry_window = $pa_config->{"event_expiry_window"}; - if ($expiry_time > 0 && $expiry_window > 0 && $expiry_window > $expiry_time) { - my $time_ref = time (); - my $expiry_limit = $time_ref - $expiry_time; - my $expiry_window = $time_ref - $expiry_window; - db_do ($dbh, 'UPDATE tevento SET estado=1, ack_utimestamp=? WHERE estado=0 AND utimestamp < ? AND utimestamp > ?', $time_ref, $expiry_limit, $expiry_window); - } - } + # Update forced alerts + pandora_exec_forced_alerts ($pa_config, $dbh); } - - # COMMON TASKS (master and non-master) - # --------------------------------------------------------------- - - # Rotate Log File + + # TASKS EXECUTED EVERY 30 SECONDS (Mid latency tasks) + # --------------------------------------------------- if (($counter % 30) == 0) { - pandora_rotate_logfile($pa_config); - - # Set event storm protection - pandora_set_event_storm_protection (pandora_get_tconfig_token ($dbh, 'event_storm_protection', 0)); - } - # Pandora self monitoring - if (defined($pa_config->{"self_monitoring"}) - && $pa_config->{"self_monitoring"} == 1 - && !is_metaconsole($pa_config) - && $counter % $pa_config->{'self_monitoring_interval'} == 0) { - pandora_self_monitoring ($pa_config, $dbh); - } - # Avoid counter overflow - if ($counter > 10000){ - $counter = 0; + # Update module status and fired alert counts + my @agents = get_db_rows ($dbh, 'SELECT id_agente, nombre, update_module_count, update_alert_count, update_secondary_groups FROM tagente WHERE disabled = 0 AND (update_module_count=1 OR update_alert_count=1 OR update_secondary_groups=1)'); + foreach my $agent (@agents) { + logger ($pa_config, "Updating module status and fired alert counts for agent " . $agent->{'nombre'}, 10); + + if ($agent->{'update_module_count'} == 1) { + pandora_update_agent_module_count ($pa_config, $dbh, $agent->{'id_agente'}); + } + + if ($agent->{'update_alert_count'} == 1) { + pandora_update_agent_alert_count ($pa_config, $dbh, $agent->{'id_agente'}); + } + + if ($agent->{'update_secondary_groups'} == 1) { + pandora_update_secondary_groups_cache ($pa_config, $dbh, $agent->{'id_agente'}); + } + } + + # Keepalive module control.(very DB intensive, not run frecuently + pandora_module_keep_alive_nd ($pa_config, $dbh); + + # Set the status of unknown modules + pandora_module_unknown ($pa_config, $dbh); + + # Check if an autodisabled agent needs to be autodisable + pandora_disable_autodisable_agents ($pa_config, $dbh); } - else { - $counter++; + + # TASKS EXECUTED EVERY 60 SECONDS (High latency tasks) + # ---------------------------------------------------- + if (($counter % 60) == 0) { + # Downtimes are executed only 30 x Server Threshold secs + pandora_planned_downtime ($pa_config, $dbh); + + # Realtime stats (Only master server!) - ( VERY HEAVY !) + # Realtimestats == 1, generated by WEB Console, not by server! + if (defined($pa_config->{"realtimestats"}) && $pa_config->{"realtimestats"} == 0){ + + # Check if I need to refresh stats + my $last_execution_stats = get_db_value ($dbh, "SELECT MAX(utimestamp) FROM tgroup_stat"); + if (!defined($last_execution_stats) || $last_execution_stats < (time() - $pa_config->{"stats_interval"})){ + pandora_group_statistics ($pa_config, $dbh); + pandora_server_statistics ($pa_config, $dbh); + } + } + + # Event auto-expiry + my $expiry_time = $pa_config->{"event_expiry_time"}; + my $expiry_window = $pa_config->{"event_expiry_window"}; + if ($expiry_time > 0 && $expiry_window > 0 && $expiry_window > $expiry_time) { + my $time_ref = time (); + my $expiry_limit = $time_ref - $expiry_time; + my $expiry_window = $time_ref - $expiry_window; + db_do ($dbh, 'UPDATE tevento SET estado=1, ack_utimestamp=? WHERE estado=0 AND utimestamp < ? AND utimestamp > ?', $time_ref, $expiry_limit, $expiry_window); + } } - }; + } + + # COMMON TASKS (master and non-master) + # --------------------------------------------------------------- + + # Rotate Log File + if (($counter % 30) == 0) { + pandora_rotate_logfile($pa_config); + + # Set event storm protection + pandora_set_event_storm_protection (pandora_get_tconfig_token ($dbh, 'event_storm_protection', 0)); + } + # Pandora self monitoring + if (defined($pa_config->{"self_monitoring"}) + && $pa_config->{"self_monitoring"} == 1 + && !is_metaconsole($pa_config) + && $counter % $pa_config->{'self_monitoring_interval'} == 0) { + pandora_self_monitoring ($pa_config, $dbh); + } + + # Avoid counter overflow + if ($counter > 10000){ + $counter = 0; + } + else { + $counter++; + } sleep (1); } + + db_disconnect($dbh); } ################################################################################ @@ -525,21 +539,13 @@ sub main() { # Load enterprise module if (enterprise_load (\%Config) == 0) { + $Config{'__enterprise_enabled'} = 0; print_message (\%Config, " [*] Pandora FMS Enterprise module not available.", 1); logger (\%Config, " [*] Pandora FMS Enterprise module not available.", 1); } else { + $Config{'__enterprise_enabled'} = 1; print_message (\%Config, " [*] " . pandora_get_initial_product_name() . " Enterprise module loaded.", 1); logger (\%Config, " [*] " . pandora_get_initial_product_name() . " Enterprise module loaded.", 1); - - if($Config{'policy_manager'} == 1) { - # Start thread to patrol policy queue - threads->create('pandora_process_policy_queue', (\%Config))->detach(); - } - - if($Config{'event_replication'} == 1) { - # Start thread to process event replication - threads->create('pandora_process_event_replication', (\%Config))->detach(); - } } # Save the start time for warmup intervals. @@ -559,9 +565,6 @@ sub main() { pandora_event (\%Config, "Warmup mode for events started.", 0, 0, 0, 0, 0, 'system', 0, $DBH); } - # Start thread to execute server tasks on the master server - threads->create('pandora_server_tasks', (\%Config))->detach(); - # Generate 'going up' events foreach my $server (@Servers) { $server->upEvent (); @@ -612,6 +615,9 @@ sub main() { $server->update(); } + # Make sure all server threads are running. + die("Server thread crashed.") unless (check_server_threads() == 1); + db_do ($DBH, "UPDATE tserver SET status = 0 WHERE UNIX_TIMESTAMP(now())-UNIX_TIMESTAMP(keepalive) > 2*server_keepalive" diff --git a/pandora_server/lib/PandoraFMS/Core.pm b/pandora_server/lib/PandoraFMS/Core.pm index 27f7794b8a..6e777c7783 100644 --- a/pandora_server/lib/PandoraFMS/Core.pm +++ b/pandora_server/lib/PandoraFMS/Core.pm @@ -4557,7 +4557,7 @@ sub pandora_process_event_replication ($) { logger($pa_config, "Starting replication events process.", 1); - while(1) { + while($THRRUN == 1) { # If we are not the master server sleep and check again. if (pandora_is_master($pa_config) == 0) { @@ -4569,6 +4569,8 @@ sub pandora_process_event_replication ($) { sleep ($replication_interval); enterprise_hook('pandora_replicate_copy_events',[$pa_config, $dbh, $dbh_metaconsole, $metaconsole_server_id, $replication_mode]); } + + db_disconnect($dbh); } ########################################################################## @@ -4588,7 +4590,7 @@ sub pandora_process_policy_queue ($) { logger($pa_config, "Starting policy queue patrol process.", 1); - while(1) { + while($THRRUN == 1) { # If we are not the master server sleep and check again. if (pandora_is_master($pa_config) == 0) { @@ -4615,7 +4617,9 @@ sub pandora_process_policy_queue ($) { } enterprise_hook('pandora_finish_queue_operation', [$dbh, $operation->{'id'}]); - } + } + + db_disconnect($dbh); } ########################################################################## diff --git a/pandora_server/lib/PandoraFMS/Tools.pm b/pandora_server/lib/PandoraFMS/Tools.pm index 896b6c3f35..41bf481198 100755 --- a/pandora_server/lib/PandoraFMS/Tools.pm +++ b/pandora_server/lib/PandoraFMS/Tools.pm @@ -77,6 +77,7 @@ our @EXPORT = qw( MODULE_WARNING MODULE_UNKNOWN MODULE_NOTINIT + $THRRUN api_call_url cron_get_closest_in_range cron_next_execution @@ -115,6 +116,9 @@ our @EXPORT = qw( valid_regex set_file_permissions uri_encode + check_server_threads + start_server_thread + stop_server_threads ); # ID of the different servers @@ -307,6 +311,12 @@ while (my ($ent, $chr) = each(%ENT2CHR)) { $CHR2ENT{$chr} = "&" . $ent . ";"; } +# Threads started by the Pandora FMS Server. +my @ServerThreads; + +# Keep threads running. +our $THRRUN :shared = 1; + ############################################################################### # Sets user:group owner for the given file ############################################################################### @@ -1740,6 +1750,48 @@ sub api_call_url { return undef; } +################################################################################ +# Start a server thread and keep track of it. +################################################################################ +sub start_server_thread { + my ($fn, $args) = @_; + + # Signal the threads to run. + $THRRUN = 1; + + my $thr = threads->create($fn, @{$args}); + push(@ServerThreads, $thr); +} + +################################################################################ +# Check the status of server threads. Returns 1 if all all running, 0 otherwise. +################################################################################ +sub check_server_threads { + my ($fn, $args) = @_; + + foreach my $thr (@ServerThreads) { + return 0 unless $thr->is_running(); + } + + return 1; +} + +################################################################################ +# Stop all server threads. +################################################################################ +sub stop_server_threads { + my ($fn, $args) = @_; + + # Signal the threads to exits. + $THRRUN = 0; + + foreach my $thr (@ServerThreads) { + $thr->detach(); + } + + @ServerThreads = (); +} + # End of function declaration # End of defined Code