diff --git a/pandora_server/ChangeLog b/pandora_server/ChangeLog index aa3543ce47..f8910a4cb2 100644 --- a/pandora_server/ChangeLog +++ b/pandora_server/ChangeLog @@ -1,3 +1,21 @@ +2009-10-07 Ramon Novoa + + * lib/PandoraFMS/SNMPServer.pm, + lib/PandoraFMS/DataServer.pm: Reverted the way enterprise functions + are called. + + * lib/PandoraFMS/Server.pm + lib/PandoraFMS/ProducerConsumerServer.pm: Handle fatal errors and + save error messages (useful for console event generation). + + * lib/PandoraFMS/Tools.pm: Improved enterprise_hook. + + * lib/PandoraFMS/Core.pm: Merged pandora_event_status into + pandora_event. + + * bin/pandora_server, lib/PandoraFMS/Config.pm: Added restart on error + and auto-restart features. + 2009-09-30 Sancho Lerena * lib/PandoraFMS/DataServer.pm: Reduced timeout for sleep diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index 71eb6f4ee8..bf39789276 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -46,6 +46,7 @@ sub pandora_shutdown () { # Stop servers foreach my $server (@Servers) { + $server->downEvent (); $server->stop (); } @@ -54,6 +55,64 @@ sub pandora_shutdown () { exit (0); } +######################################################################################## +# Server startup. +######################################################################################## +sub pandora_startup () { + + # Connect to the DB + $DBH = db_connect ('mysql', $Config{'dbname'}, $Config{'dbhost'}, 3306, + $Config{'dbuser'}, $Config{'dbpass'}); + + pandora_audit (\%Config, 'Pandora FMS Server Daemon starting', 'SYSTEM', 'System', $DBH); + + # Start logging + pandora_start_log (\%Config); + + # Load servers + pandora_reset_server (\%Config, $DBH); + push (@Servers, new PandoraFMS::DataServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::NetworkServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::ReconServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::SNMPServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::WMIServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::PluginServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH)); + + enterprise_hook('load_enterprise_servers', [\@Servers, \%Config, $DBH]); + + # Remove disabled servers + @Servers = grep { defined ($_) } @Servers; + + # Run + foreach my $server (@Servers) { + $server->run (); + } +} + +######################################################################################## +# Server restart. +######################################################################################## +sub pandora_restart () { + + # Stop the servers + foreach my $server (@Servers) { + $server->stop (); + } + + # Remove the servers + while (pop (@Servers)) {}; + + # Close STDERR, redirected by pandora_start_log + close (STDERR); + + # Wait before trying to start again + sleep ($Config{'restart_delay'}); + + # Start the servers + pandora_startup (); +} + ######################################################################################## # Server crash. Handler to write in the log unhandled errors and write it to console ######################################################################################## @@ -83,8 +142,9 @@ pandora_init(\%Config, 'Pandora FMS Server'); pandora_load_config (\%Config); # Load enterprise module -$Config{"enterprise"} = enterprise_load (); -if ($Config{"enterprise"} == 1){ +if (enterprise_load () == 0) { + print " [*] Pandora FMS Enterprise module not available.\n"; +} else { print " [*] Pandora FMS Enterprise module loaded.\n"; } @@ -94,55 +154,53 @@ if ($Config{'daemon'} == 1) { pandora_daemonize (\%Config); } -# Connect to the DB -$DBH = db_connect ('mysql', $Config{'dbname'}, $Config{'dbhost'}, 3306, - $Config{'dbuser'}, $Config{'dbpass'}); +# Start the servers +pandora_startup (); -pandora_audit (\%Config, 'Pandora FMS Server Daemon starting', 'SYSTEM', 'System', $DBH); - -# Start logging -pandora_start_log (\%Config); - -# Load servers -pandora_reset_server (\%Config, $DBH); -push (@Servers, new PandoraFMS::DataServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::NetworkServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::ReconServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::SNMPServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::WMIServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::PluginServer (\%Config, $DBH)); -push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH)); - -if ($Config{"enterprise"} == 1){ - enterprise_hook('load_enterprise_servers', [\@Servers, \%Config, $DBH]); -} - -# Remove disabled servers -@Servers = grep { defined ($_) } @Servers; - -# Run +# Generate 'going up' events foreach my $server (@Servers) { - $server->run (); $server->upEvent (); } # Main loop +my $time_ref = time (); while (1) { - - # Update server status - foreach my $server (@Servers) { - pandora_shutdown () unless ($server->checkThreads () == 1); - $server->update (); - } - eval { + eval { + + # Update server status + foreach my $server (@Servers) { + die ($server->getErrStr ()) unless ($server->checkThreads () == 1); + $server->update (); + } + pandora_planned_downtime (\%Config, $DBH); pandora_exec_forced_alerts (\%Config, $DBH); pandora_module_keep_alive_nd (\%Config, $DBH); }; - + + # Restart on error or auto restart if ($@) { - pandora_shutdown (); + + # Generate 'restarting' events + foreach my $server (@Servers) { + $server->restartEvent ($@); + } + + logger (\%Config, 'Pandora FMS Server restarting (' . $@ . ') in ' . $Config{'restart_delay'} . ' seconds.', 0); + pandora_restart (); + } elsif (($Config{'auto_restart'} > 0) && (time () - $time_ref > $Config{'auto_restart'})) { + $time_ref = time (); + + # Mute + open(OLDOUT, ">&STDOUT"); + open (STDOUT, '>/dev/null'); + + # Restart + pandora_restart (); + + # Unmute + open(STDOUT, ">&OLDOUT"); } threads->yield; diff --git a/pandora_server/lib/PandoraFMS/Config.pm b/pandora_server/lib/PandoraFMS/Config.pm index cd0b7df19d..67059c45da 100644 --- a/pandora_server/lib/PandoraFMS/Config.pm +++ b/pandora_server/lib/PandoraFMS/Config.pm @@ -226,6 +226,12 @@ sub pandora_load_config { # Ignore the timestamp in the XML and use the file timestamp instead $pa_config->{'use_xml_timestamp'} = 0; + # Server restart delay in seconds + $pa_config->{'restart_delay'} = 60; + + # Auto restart every x seconds + $pa_config->{'auto_restart'} = 0; + # Check for UID0 if ($pa_config->{"quiet"} != 0){ if ($> == 0){ @@ -491,6 +497,12 @@ sub pandora_load_config { elsif ($parametro =~ m/^use_xml_timestamp\s([0-1])/i) { $pa_config->{'use_xml_timestamp'} = clean_blank($1); } + elsif ($parametro =~ m/^restart_delay\s+(\d+)/i) { + $pa_config->{'restart_delay'} = clean_blank($1); + } + elsif ($parametro =~ m/^auto_restart\s+(\d+)/i) { + $pa_config->{'auto_restart'} = clean_blank($1); + } } # end of loop for parameter # diff --git a/pandora_server/lib/PandoraFMS/Core.pm b/pandora_server/lib/PandoraFMS/Core.pm index 80ba7d2391..fe91098fc8 100644 --- a/pandora_server/lib/PandoraFMS/Core.pm +++ b/pandora_server/lib/PandoraFMS/Core.pm @@ -44,7 +44,6 @@ our @EXPORT = qw( pandora_evaluate_compound_alert pandora_evaluate_snmp_alerts pandora_event - pandora_event_status pandora_execute_alert pandora_execute_action pandora_exec_forced_alerts @@ -784,21 +783,6 @@ sub pandora_event (%$$$$$$$$) { VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?)', $id_agente, $id_grupo, $evento, $timestamp, $utimestamp, $event_type, $id_agentmodule, $id_alert_am, $severity); } -########################################################################## -# Generate an event with the given status. TODO: Merge with pandora_event -########################################################################## -sub pandora_event_status ($$$$$$$$$$) { - my ($pa_config, $evento, $id_grupo, $id_agente, $severity, - $id_alert_am, $id_agentmodule, $event_type, $status, $dbh) = @_; - - my $utimestamp = time(); - my $timestamp = strftime ("%Y-%m-%d %H:%M:%S", localtime($utimestamp)); - $id_agentmodule = 0 unless defined ($id_agentmodule); - - db_do ($dbh, 'INSERT INTO tevento (`id_agente`, `id_grupo`, `evento`, `timestamp`, `estado`, `utimestamp`, `event_type`, `id_agentmodule`, `id_alert_am`, `criticity`) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', $id_agente, $id_grupo, $evento, $timestamp, $status, $utimestamp, $event_type, $id_agentmodule, $id_alert_am, $severity); -} - ########################################################################## # Update module status on error. ########################################################################## @@ -1147,8 +1131,8 @@ sub generate_status_event ($$$$$$$) { } # Generate the event - pandora_event_status ($pa_config, $description, $agent->{'id_grupo'}, $module->{'id_agente'}, - $severity, 0, $module->{'id_agente_modulo'}, $event_type, $status, $dbh); + pandora_event ($pa_config, $description, $agent->{'id_grupo'}, $module->{'id_agente'}, + $severity, 0, $module->{'id_agente_modulo'}, $event_type, $dbh); } ########################################################################## diff --git a/pandora_server/lib/PandoraFMS/DataServer.pm b/pandora_server/lib/PandoraFMS/DataServer.pm index a7543db131..3a36d7858a 100644 --- a/pandora_server/lib/PandoraFMS/DataServer.pm +++ b/pandora_server/lib/PandoraFMS/DataServer.pm @@ -33,9 +33,6 @@ use PandoraFMS::DB; use PandoraFMS::Core; use PandoraFMS::ProducerConsumerServer; -# Load enterprise module -enterprise_load (); - # Inherits from PandoraFMS::ProducerConsumerServer our @ISA = qw(PandoraFMS::ProducerConsumerServer); @@ -170,7 +167,7 @@ sub process_xml_data ($$$$) { } # Check some variables - $interval = 300 unless defined ($interval); + $interval = 300 if (! defined ($interval) || $interval eq ''); $os_version = 'N/A' if (! defined ($os_version) || $os_version eq ''); # Get agent id @@ -234,10 +231,8 @@ sub process_xml_data ($$$$) { } # Process inventory modules - if ($pa_config->{enterprise} == 1){ - enterprise_hook('process_inventory_data', [$pa_config, $data, $server_id, $agent_name, + enterprise_hook('process_inventory_data', [$pa_config, $data, $server_id, $agent_name, $interval, $timestamp, $dbh]); - } } ########################################################################## diff --git a/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm b/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm index 3aa482edf9..af537f609c 100644 --- a/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm +++ b/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm @@ -99,37 +99,43 @@ sub data_producer ($$$$$) { my ($self, $task_queue, $pending_tasks, $sem, $task_sem) = @_; my $pa_config = $self->getConfig (); - # Connect to the DB - my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306, - $pa_config->{'dbuser'}, $pa_config->{'dbpass'}); - $self->setDBH ($dbh); + eval { + # Connect to the DB + my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306, + $pa_config->{'dbuser'}, $pa_config->{'dbpass'}); + $self->setDBH ($dbh); - while (1) { + while (1) { - # Get pending tasks - my @tasks = &{$self->{'_producer'}}($self); - - # Update queue size for statistics - $self->setQueueSize (scalar @{$task_queue}); - - foreach my $task (@tasks) { - $sem->down; + # Get pending tasks + my @tasks = &{$self->{'_producer'}}($self); - if (defined $pending_tasks->{$task}) { - $sem->up; - next; - } + # Update queue size for statistics + $self->setQueueSize (scalar @{$task_queue}); + + foreach my $task (@tasks) { + $sem->down; - # Queue task and signal consumers - $pending_tasks->{$task} = 0; - push (@{$task_queue}, $task); - $task_sem->up; - - $sem->up; - } + if (defined $pending_tasks->{$task}) { + $sem->up; + next; + } + + # Queue task and signal consumers + $pending_tasks->{$task} = 0; + push (@{$task_queue}, $task); + $task_sem->up; + + $sem->up; + } - threads->yield; - sleep ($pa_config->{'server_threshold'}); + threads->yield; + sleep ($pa_config->{'server_threshold'}); + } + }; + + if ($@) { + $self->setErrStr ($@); } } @@ -140,29 +146,35 @@ sub data_consumer ($$$$$) { my ($self, $task_queue, $pending_tasks, $sem, $task_sem) = @_; my $pa_config = $self->getConfig (); - # Connect to the DB - my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306, - $pa_config->{'dbuser'}, $pa_config->{'dbpass'}); - $self->setDBH ($dbh); + eval { + # Connect to the DB + my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306, + $pa_config->{'dbuser'}, $pa_config->{'dbpass'}); + $self->setDBH ($dbh); - while (1) { + while (1) { - # Wait for data - $task_sem->down; + # Wait for data + $task_sem->down; - $sem->down; - my $task = shift (@{$task_queue}); - $sem->up; + $sem->down; + my $task = shift (@{$task_queue}); + $sem->up; - # Execute task - &{$self->{'_consumer'}}($self, $task); + # Execute task + &{$self->{'_consumer'}}($self, $task); - # Update task status - $sem->down; - delete ($pending_tasks->{$task}); - $sem->up; + # Update task status + $sem->down; + delete ($pending_tasks->{$task}); + $sem->up; - threads->yield; + threads->yield; + } + }; + + if ($@) { + $self->setErrStr ($@); } } diff --git a/pandora_server/lib/PandoraFMS/SNMPServer.pm b/pandora_server/lib/PandoraFMS/SNMPServer.pm index 24ef787d38..61192d6ba4 100644 --- a/pandora_server/lib/PandoraFMS/SNMPServer.pm +++ b/pandora_server/lib/PandoraFMS/SNMPServer.pm @@ -32,9 +32,6 @@ use PandoraFMS::DB; use PandoraFMS::Core; use PandoraFMS::Server; -# Load enterprise module -enterprise_load (); - # Inherits from PandoraFMS::Server our @ISA = qw(PandoraFMS::Server); @@ -107,12 +104,7 @@ sub pandora_snmptrapd { # Skip already processed lines readline SNMPLOGFILE for (1..$last_line); - - my $trap2agent = 0; - - if ($pa_config->{enterprise} == 1){ - $trap2agent = enterprise_hook('snmp_get_trap2agent', [$dbh]); - } + my $trap2agent = enterprise_hook('snmp_get_trap2agent', [$dbh]); # Main loop while (1) { diff --git a/pandora_server/lib/PandoraFMS/Server.pm b/pandora_server/lib/PandoraFMS/Server.pm index c4f74593e3..f2372bfae4 100644 --- a/pandora_server/lib/PandoraFMS/Server.pm +++ b/pandora_server/lib/PandoraFMS/Server.pm @@ -42,10 +42,12 @@ sub new ($$$;$) { _num_threads => 1, _threads => [], _queue_size => 0, + _errstr => '' }; # Share variables that may be set from different threads share ($self->{'_queue_size'}); + share ($self->{'_errstr'}); # Thread kill signal handler #$SIG{'KILL'} = sub { @@ -167,6 +169,24 @@ sub getServerType ($) { return $self->{'_server_type'}; } +######################################################################################## +# Set error string. +######################################################################################## +sub setErrStr ($$) { + my ($self, $errstr) = @_; + + $self->{'_errstr'} = $errstr; +} + +######################################################################################## +# Get error string. +######################################################################################## +sub getErrStr ($) { + my $self = shift; + + return $self->{'_errstr'}; +} + ######################################################################################## # Add a thread to the server thread list. ######################################################################################## @@ -214,6 +234,18 @@ sub downEvent ($) { 0, 0, 4, 0, 0, 'system', $self->{'_dbh'}); } +######################################################################################## +# Generate a 'restarting' event. +######################################################################################## +sub restartEvent ($$) { + my ($self, $msg) = @_; + + return unless defined ($self->{'_dbh'}); + pandora_event ($self->{'_pa_config'}, $self->{'_pa_config'}->{'servername'} . + $ServerTypes[$self->{'_server_type'}] . " RESTARTING ($msg)", + 0, 0, 4, 0, 0, 'system', $self->{'_dbh'}); +} + ######################################################################################## # Update server status. ######################################################################################## @@ -236,9 +268,6 @@ sub stop ($) { # Update server status pandora_update_server ($self->{'_pa_config'}, $self->{'_dbh'}, $self->{'_pa_config'}->{'servername'}, 0, $self->{'_server_type'}, 0, 0); - - # Generate an event - $self->downEvent (); }; # Kill server threads diff --git a/pandora_server/lib/PandoraFMS/Tools.pm b/pandora_server/lib/PandoraFMS/Tools.pm index cc73addecc..4c19bfdd6c 100644 --- a/pandora_server/lib/PandoraFMS/Tools.pm +++ b/pandora_server/lib/PandoraFMS/Tools.pm @@ -21,6 +21,7 @@ use warnings; use Time::Local; use POSIX qw(setsid strftime); use Mail::Sendmail; # New in 2.0. Used to sendmail internally, without external scripts +use Module::Loaded; require Exporter; @@ -334,7 +335,14 @@ sub float_equal { # enterprise_hook (). ########################################################################## sub enterprise_load () { - eval 'use PandoraFMS::Enterprise;'; + + # Already loaded + return 1 if (is_loaded ('PandoraFMS::Enterprise')); + + # Try to load the module + eval 'local $SIG{__DIE__}; require PandoraFMS::Enterprise;'; + + # Ops return 0 if ($@); return 1; } @@ -351,6 +359,7 @@ sub enterprise_hook ($$) { # Prepend the package name $func = 'PandoraFMS::Enterprise::' . $func; + return undef unless (defined (&$func)); # Try to call the function my $output = eval { &$func (@args); };