2009-10-07 Ramon Novoa <rnovoa@artica.es>

* lib/PandoraFMS/SNMPServer.pm,
          lib/PandoraFMS/DataServer.pm: Reverted the way enterprise functions
          are called.

        * lib/PandoraFMS/Server.pm
          lib/PandoraFMS/ProducerConsumerServer.pm: Handle fatal errors and
          save error messages (useful for console event generation).

        * lib/PandoraFMS/Tools.pm: Improved enterprise_hook.

        * lib/PandoraFMS/Core.pm: Merged pandora_event_status into
          pandora_event.

        * bin/pandora_server, lib/PandoraFMS/Config.pm: Added restart on error
          and auto-restart features.




git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@2010 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f

pandora_server/ChangeLog

@@ -1,3 +1,21 @@
+2009-10-07  Ramon Novoa  <rnovoa@artica.es>
+
+	* lib/PandoraFMS/SNMPServer.pm,
+	  lib/PandoraFMS/DataServer.pm: Reverted the way enterprise functions
+	  are called.
+
+	* lib/PandoraFMS/Server.pm
+	  lib/PandoraFMS/ProducerConsumerServer.pm: Handle fatal errors and
+	  save error messages (useful for console event generation).
+
+	* lib/PandoraFMS/Tools.pm: Improved enterprise_hook.
+
+	* lib/PandoraFMS/Core.pm: Merged pandora_event_status into
+	  pandora_event.
+
+	* bin/pandora_server, lib/PandoraFMS/Config.pm: Added restart on error
+	  and auto-restart features.
+
 2009-09-30  Sancho Lerena <slerena@artica.es>
 
 	* lib/PandoraFMS/DataServer.pm: Reduced timeout for sleep

pandora_server/bin/pandora_server

@@ -46,6 +46,7 @@ sub pandora_shutdown () {
 
 	# Stop servers
 	foreach my $server (@Servers) {
+		$server->downEvent ();
 		$server->stop ();
 	}
 
@@ -54,6 +55,64 @@ sub pandora_shutdown () {
 	exit (0);
 }
 
+########################################################################################
+# Server startup.
+########################################################################################
+sub pandora_startup () {
+
+	# Connect to the DB
+	$DBH = db_connect ('mysql', $Config{'dbname'}, $Config{'dbhost'}, 3306,
+					   $Config{'dbuser'}, $Config{'dbpass'});
+
+	pandora_audit (\%Config, 'Pandora FMS Server Daemon starting', 'SYSTEM', 'System', $DBH);
+
+	# Start logging
+	pandora_start_log (\%Config);
+
+	# Load servers
+	pandora_reset_server (\%Config, $DBH);
+	push (@Servers, new PandoraFMS::DataServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::NetworkServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::ReconServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::SNMPServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::WMIServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::PluginServer (\%Config, $DBH));
+	push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH));
+
+	enterprise_hook('load_enterprise_servers', [\@Servers, \%Config, $DBH]);
+
+	# Remove disabled servers
+	@Servers = grep { defined ($_) } @Servers;
+
+	# Run
+	foreach my $server (@Servers) {
+		$server->run ();
+	}
+}
+
+########################################################################################
+# Server restart.
+########################################################################################
+sub pandora_restart () {
+
+	# Stop the servers
+	foreach my $server (@Servers) {
+		$server->stop ();
+	}
+
+	# Remove the servers
+	while (pop (@Servers)) {};
+	
+	# Close STDERR, redirected by pandora_start_log
+	close (STDERR);
+
+	# Wait before trying to start again
+	sleep ($Config{'restart_delay'});
+
+	# Start the servers
+	pandora_startup ();
+}
+
 ########################################################################################
 # Server crash. Handler to write in the log unhandled errors and write it to console
 ########################################################################################
@@ -83,8 +142,9 @@ pandora_init(\%Config, 'Pandora FMS Server');
 pandora_load_config (\%Config);
 
 # Load enterprise module
-$Config{"enterprise"} = enterprise_load ();
+if (enterprise_load () == 0) {
-if ($Config{"enterprise"} == 1){
+	print " [*] Pandora FMS Enterprise module not available.\n";
+} else {
 	print " [*] Pandora FMS Enterprise module loaded.\n";
 }
 
@@ -94,55 +154,53 @@ if ($Config{'daemon'} == 1) {
 	pandora_daemonize (\%Config);
 }
 
-# Connect to the DB
+# Start the servers
-$DBH = db_connect ('mysql', $Config{'dbname'}, $Config{'dbhost'}, 3306,
+pandora_startup ();
-                   $Config{'dbuser'}, $Config{'dbpass'});
 
-pandora_audit (\%Config, 'Pandora FMS Server Daemon starting', 'SYSTEM', 'System', $DBH);
+# Generate 'going up' events
-
-# Start logging
-pandora_start_log (\%Config);
-
-# Load servers
-pandora_reset_server (\%Config, $DBH);
-push (@Servers, new PandoraFMS::DataServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::NetworkServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::ReconServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::SNMPServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::WMIServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::PluginServer (\%Config, $DBH));
-push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH));
-
-if ($Config{"enterprise"} == 1){
-	enterprise_hook('load_enterprise_servers', [\@Servers, \%Config, $DBH]);
-}
-
-# Remove disabled servers
-@Servers = grep { defined ($_) } @Servers;
-
-# Run
 foreach my $server (@Servers) {
-	$server->run ();
 	$server->upEvent ();
 }
 
 # Main loop
+my $time_ref = time ();
 while (1) {
 
+	eval {	
+		
 		# Update server status
 		foreach my $server (@Servers) {
-		pandora_shutdown () unless ($server->checkThreads () == 1);
+			die ($server->getErrStr ()) unless ($server->checkThreads () == 1);
 			$server->update ();
 		}
 
-	eval {
 		pandora_planned_downtime (\%Config, $DBH);
 		pandora_exec_forced_alerts (\%Config, $DBH);
 		pandora_module_keep_alive_nd (\%Config, $DBH);
 	};
 
+	# Restart on error or auto restart
 	if ($@) {
-		pandora_shutdown ();
+
+		# Generate 'restarting' events
+		foreach my $server (@Servers) {
+			$server->restartEvent ($@);
+		}
+
+		logger (\%Config, 'Pandora FMS Server restarting (' . $@ . ') in ' . $Config{'restart_delay'} . ' seconds.', 0);
+		pandora_restart ();
+	} elsif (($Config{'auto_restart'} > 0) && (time () - $time_ref > $Config{'auto_restart'})) {
+		$time_ref = time ();
+
+		# Mute
+		open(OLDOUT, ">&STDOUT");
+		open (STDOUT, '>/dev/null');
+
+		# Restart
+		pandora_restart ();
+		
+		# Unmute
+		open(STDOUT, ">&OLDOUT");
 	}
 
 	threads->yield;

pandora_server/lib/PandoraFMS/Config.pm

@@ -226,6 +226,12 @@ sub pandora_load_config {
 	# Ignore the timestamp in the XML and use the file timestamp instead
     $pa_config->{'use_xml_timestamp'} = 0; 
 
+	# Server restart delay in seconds
+    $pa_config->{'restart_delay'} = 60; 
+
+	# Auto restart every x seconds
+    $pa_config->{'auto_restart'} = 0; 
+
 	# Check for UID0
     if ($pa_config->{"quiet"} != 0){
 	    if ($> == 0){
@@ -491,6 +497,12 @@ sub pandora_load_config {
 		elsif ($parametro =~ m/^use_xml_timestamp\s([0-1])/i) {
 			$pa_config->{'use_xml_timestamp'} = clean_blank($1);
 		}
+		elsif ($parametro =~ m/^restart_delay\s+(\d+)/i) {
+                        $pa_config->{'restart_delay'} = clean_blank($1);
+		}
+		elsif ($parametro =~ m/^auto_restart\s+(\d+)/i) {
+                        $pa_config->{'auto_restart'} = clean_blank($1);
+		}
 	} # end of loop for parameter #
 
 

pandora_server/lib/PandoraFMS/Core.pm

@@ -44,7 +44,6 @@ our @EXPORT = qw(
 	pandora_evaluate_compound_alert
 	pandora_evaluate_snmp_alerts
 	pandora_event
-	pandora_event_status
 	pandora_execute_alert
 	pandora_execute_action
 	pandora_exec_forced_alerts
@@ -784,21 +783,6 @@ sub pandora_event (%$$$$$$$$) {
 	              VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?)', $id_agente, $id_grupo, $evento, $timestamp, $utimestamp, $event_type, $id_agentmodule, $id_alert_am, $severity);
 }
 
-##########################################################################
-# Generate an event with the given status. TODO: Merge with pandora_event
-##########################################################################
-sub pandora_event_status ($$$$$$$$$$) {
-	my ($pa_config, $evento, $id_grupo, $id_agente, $severity,
-		$id_alert_am, $id_agentmodule, $event_type, $status, $dbh) = @_;
-
-	my $utimestamp = time();
-	my $timestamp = strftime ("%Y-%m-%d %H:%M:%S", localtime($utimestamp));
-	$id_agentmodule = 0 unless defined ($id_agentmodule);
-
-	db_do ($dbh, 'INSERT INTO tevento (`id_agente`, `id_grupo`, `evento`, `timestamp`, `estado`, `utimestamp`, `event_type`, `id_agentmodule`, `id_alert_am`, `criticity`)
-	              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', $id_agente, $id_grupo, $evento, $timestamp, $status, $utimestamp, $event_type, $id_agentmodule, $id_alert_am, $severity);
-}
-
 ##########################################################################
 # Update module status on error.
 ##########################################################################
@@ -1147,8 +1131,8 @@ sub generate_status_event ($$$$$$$) {
 	}
 
 	# Generate the event
-	pandora_event_status ($pa_config, $description, $agent->{'id_grupo'}, $module->{'id_agente'},
+	pandora_event ($pa_config, $description, $agent->{'id_grupo'}, $module->{'id_agente'},
-	               $severity, 0, $module->{'id_agente_modulo'}, $event_type, $status, $dbh);
+	               $severity, 0, $module->{'id_agente_modulo'}, $event_type, $dbh);
 }
 
 ##########################################################################

pandora_server/lib/PandoraFMS/DataServer.pm

@@ -33,9 +33,6 @@ use PandoraFMS::DB;
 use PandoraFMS::Core;
 use PandoraFMS::ProducerConsumerServer;
 
-# Load enterprise module
-enterprise_load ();
-
 # Inherits from PandoraFMS::ProducerConsumerServer
 our @ISA = qw(PandoraFMS::ProducerConsumerServer);
 
@@ -170,7 +167,7 @@ sub process_xml_data ($$$$) {
 	}
 
   	# Check some variables
-   	$interval = 300 unless defined ($interval);
+   	$interval = 300 if (! defined ($interval) || $interval eq '');
    	$os_version = 'N/A' if (! defined ($os_version) || $os_version eq '');
   
   	# Get agent id
@@ -234,11 +231,9 @@ sub process_xml_data ($$$$) {
 	}
 
 	# Process inventory modules
-	if ($pa_config->{enterprise} == 1){
 	enterprise_hook('process_inventory_data', [$pa_config, $data, $server_id, $agent_name,
 	                                           $interval, $timestamp, $dbh]);
 }
-}
 
 ##########################################################################
 # Process module data, creating module if necessary.

pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm

@@ -99,6 +99,7 @@ sub data_producer ($$$$$) {
 	my ($self, $task_queue, $pending_tasks, $sem, $task_sem) = @_;
 	my $pa_config = $self->getConfig ();
 
+	eval {
 		# Connect to the DB
 		my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306,
 							  $pa_config->{'dbuser'}, $pa_config->{'dbpass'});
@@ -131,6 +132,11 @@ sub data_producer ($$$$$) {
 			threads->yield;
 			sleep ($pa_config->{'server_threshold'});
 		}
+	};
+	
+	if ($@) {
+		$self->setErrStr ($@);
+	}
 }
 
 ###############################################################################
@@ -140,6 +146,7 @@ sub data_consumer ($$$$$) {
 	my ($self, $task_queue, $pending_tasks, $sem, $task_sem) = @_;
 	my $pa_config = $self->getConfig ();
 
+	eval {
 		# Connect to the DB
 		my $dbh = db_connect ('mysql', $pa_config->{'dbname'}, $pa_config->{'dbhost'}, 3306,
 							  $pa_config->{'dbuser'}, $pa_config->{'dbpass'});
@@ -164,6 +171,11 @@ sub data_consumer ($$$$$) {
 
 			threads->yield;
 		}
+	};
+
+	if ($@) {
+		$self->setErrStr ($@);
+	}
 }
 
 1;

pandora_server/lib/PandoraFMS/SNMPServer.pm

@@ -32,9 +32,6 @@ use PandoraFMS::DB;
 use PandoraFMS::Core;
 use PandoraFMS::Server;
 
-# Load enterprise module
-enterprise_load ();
-
 # Inherits from PandoraFMS::Server
 our @ISA = qw(PandoraFMS::Server);
 
@@ -107,12 +104,7 @@ sub pandora_snmptrapd {
 	# Skip already processed lines
 	readline SNMPLOGFILE for (1..$last_line);
 
-	
+	my $trap2agent = enterprise_hook('snmp_get_trap2agent', [$dbh]);
-	my $trap2agent = 0;
-	
-	if ($pa_config->{enterprise} == 1){
-		$trap2agent = enterprise_hook('snmp_get_trap2agent', [$dbh]);
-	}
 
 	# Main loop
 	while (1) {

pandora_server/lib/PandoraFMS/Server.pm

@@ -42,10 +42,12 @@ sub new ($$$;$) {
         _num_threads => 1,
         _threads => [],
         _queue_size => 0,
+		_errstr => ''
     };
 
 	# Share variables that may be set from different threads
 	share ($self->{'_queue_size'});
+	share ($self->{'_errstr'});
 
 	# Thread kill signal handler
 	#$SIG{'KILL'} = sub {
@@ -167,6 +169,24 @@ sub getServerType ($) {
 	return $self->{'_server_type'};
 }
 
+########################################################################################
+# Set error string.
+########################################################################################
+sub setErrStr ($$) {
+	my ($self, $errstr) = @_;
+
+	$self->{'_errstr'} = $errstr;
+}
+
+########################################################################################
+# Get error string.
+########################################################################################
+sub getErrStr ($) {
+	my $self = shift;
+	
+	return $self->{'_errstr'};
+}
+
 ########################################################################################
 # Add a thread to the server thread list.
 ########################################################################################
@@ -214,6 +234,18 @@ sub downEvent ($) {
 	               0, 0, 4, 0, 0, 'system', $self->{'_dbh'});
 }
 
+########################################################################################
+# Generate a 'restarting' event.
+########################################################################################
+sub restartEvent ($$) {
+	my ($self, $msg) = @_;
+
+	return unless defined ($self->{'_dbh'});
+	pandora_event ($self->{'_pa_config'}, $self->{'_pa_config'}->{'servername'} .
+	               $ServerTypes[$self->{'_server_type'}] . " RESTARTING ($msg)",
+	               0, 0, 4, 0, 0, 'system', $self->{'_dbh'});
+}
+
 ########################################################################################
 # Update server status.
 ########################################################################################
@@ -236,9 +268,6 @@ sub stop ($) {
 		# Update server status
 		pandora_update_server ($self->{'_pa_config'}, $self->{'_dbh'}, $self->{'_pa_config'}->{'servername'},
 		                       0, $self->{'_server_type'}, 0, 0);
-
-		# Generate an event
-		$self->downEvent ();
 	};
 
 	# Kill server threads

pandora_server/lib/PandoraFMS/Tools.pm

@@ -21,6 +21,7 @@ use warnings;
 use Time::Local;
 use POSIX qw(setsid strftime);
 use Mail::Sendmail;	# New in 2.0. Used to sendmail internally, without external scripts
+use Module::Loaded;
 
 require Exporter;
 
@@ -334,7 +335,14 @@ sub float_equal {
 # enterprise_hook ().
 ##########################################################################
 sub enterprise_load () {
-	eval 'use PandoraFMS::Enterprise;';
+
+	# Already loaded
+	return 1 if (is_loaded ('PandoraFMS::Enterprise'));
+	
+	# Try to load the module
+	eval 'local $SIG{__DIE__}; require PandoraFMS::Enterprise;';
+	
+	# Ops
 	return 0 if ($@);
 	return 1;
 }
@@ -351,6 +359,7 @@ sub enterprise_hook ($$) {
 
 	# Prepend the package name
 	$func = 'PandoraFMS::Enterprise::' . $func;
+	return undef unless (defined (&$func));
 
 	# Try to call the function
 	my $output = eval { &$func (@args); };