Merge branch 'fix/pandora_enterprise#2593-server-signals' into 'develop'

Check and restart server threads if necessary.

See merge request artica/pandorafms!1674

pandora_server/bin/pandora_server

@@ -101,8 +101,21 @@ sub pandora_startup () {
 	# Generate the encryption key after reading the passphrase.
 	$Config{"encryption_key"} = enterprise_hook('pandora_get_encryption_key', [\%Config, $Config{"encryption_passphrase"}]);
 
-	# Update the agent cache.
+	# Kill any running server threads.
-	threads->create('enterprise_hook', ('update_agent_cache', [\%Config]))->detach() if ($Config{'node_metaconsole'} == 1);
+	stop_server_threads();
+
+	# Start the task execution thread.
+	start_server_thread(\&pandora_server_tasks, [\%Config]);
+	
+	# Start the policy queue thread.
+	start_server_thread(\&pandora_process_policy_queue, [\%Config]) if ($Config{'__enterprise_enabled'} == 1 && $Config{'policy_manager'} == 1);
+		
+	# Start the event replication thread. Do not start with start_server_thread, this thread may exit on its own.
+	threads->create(\&pandora_process_event_replication, [\%Config]) if($Config{'__enterprise_enabled'} == 1 && $Config{'event_replication'} == 1);
+
+	# Update the agent cache. Do not start with start_server_thread, this thread updates the agent cache and exits.
+	threads->create(\&enterprise_hook, ['update_agent_cache', [\%Config]])->detach() if ($Config{'node_metaconsole'} == 1);
+
 	pandora_audit (\%Config, $Config{'rb_product_name'} . ' Server Daemon starting', 'SYSTEM', 'System', $DBH);
 	
 	# Load servers
@@ -143,9 +156,11 @@ sub pandora_restart (;$) {
 	my $sleep_time = @_ > 0 ? $_[0] : $Config{'restart_delay'};
 		
 	# Stop the servers
+	eval {
 		foreach my $server (@Servers) {
 			$server->stop ();
 		}
+	};
 	
 	# Remove the servers
 	while (pop (@Servers)) {};
@@ -274,10 +289,8 @@ sub pandora_server_tasks ($) {
 	# Get the console DB connection
 	my $dbh = db_connect ($pa_config->{'dbengine'}, $pa_config->{'dbname'}, $pa_config->{'dbhost'}, $pa_config->{'dbport'},
 						$pa_config->{'dbuser'}, $pa_config->{'dbpass'});
-	
 	my $counter = 0;
-	while ($RUN == 1) {
+	while ($THRRUN == 1) {
-		eval{
 		if (pandora_is_master($pa_config) == 1) {
 
 			# TASKS EXECUTED EVERY 5 SECONDS (Low latency tasks)
@@ -375,10 +388,11 @@ sub pandora_server_tasks ($) {
 		else {
 			$counter++;
 		}
-		};
 
 		sleep (1);
 	}
+
+	db_disconnect($dbh);
 }
 
 ################################################################################
@@ -525,21 +539,13 @@ sub main() {
 	
 	# Load enterprise module
 	if (enterprise_load (\%Config) == 0) {
+		$Config{'__enterprise_enabled'} = 0;
 		print_message (\%Config, " [*] Pandora FMS Enterprise module not available.", 1);
 		logger (\%Config, " [*] Pandora FMS Enterprise module not available.", 1);
 	} else {
+		$Config{'__enterprise_enabled'} = 1;
 		print_message (\%Config, " [*] " . pandora_get_initial_product_name() . " Enterprise module loaded.", 1);
 		logger (\%Config, " [*] " . pandora_get_initial_product_name() . " Enterprise module loaded.", 1);
-	
-		if($Config{'policy_manager'} == 1) {
-			# Start thread to patrol policy queue
-			threads->create('pandora_process_policy_queue', (\%Config))->detach();
-		}
-		
-		if($Config{'event_replication'} == 1) {
-			# Start thread to process event replication
-			threads->create('pandora_process_event_replication', (\%Config))->detach();
-		}
 	}
 	
 	# Save the start time for warmup intervals.
@@ -559,9 +565,6 @@ sub main() {
 		pandora_event (\%Config, "Warmup mode for events started.", 0, 0, 0, 0, 0, 'system', 0, $DBH);
 	}
 
-	# Start thread to execute server tasks on the master server
-	threads->create('pandora_server_tasks', (\%Config))->detach();
-	
 	# Generate 'going up' events
 	foreach my $server (@Servers) {
 		$server->upEvent ();
@@ -612,6 +615,9 @@ sub main() {
 				$server->update();
 			}
 	
+			# Make sure all server threads are running.
+			die("Server thread crashed.") unless (check_server_threads() == 1);
+
 			db_do ($DBH,
 				"UPDATE tserver SET status = 0
 				WHERE UNIX_TIMESTAMP(now())-UNIX_TIMESTAMP(keepalive) > 2*server_keepalive"

pandora_server/lib/PandoraFMS/Core.pm

@@ -4558,7 +4558,7 @@ sub pandora_process_event_replication ($) {
 				
 	logger($pa_config, "Starting replication events process.", 1);
 
-	while(1) { 
+	while($THRRUN == 1) { 
 
 		# If we are not the master server sleep and check again.
 		if (pandora_is_master($pa_config) == 0) {
@@ -4570,6 +4570,8 @@ sub pandora_process_event_replication ($) {
 		sleep ($replication_interval);
 		enterprise_hook('pandora_replicate_copy_events',[$pa_config, $dbh, $dbh_metaconsole, $metaconsole_server_id, $replication_mode]);
 	}
+
+	db_disconnect($dbh);
 }
 
 ##########################################################################
@@ -4589,7 +4591,7 @@ sub pandora_process_policy_queue ($) {
 
 	logger($pa_config, "Starting policy queue patrol process.", 1);
 
-	while(1) {
+	while($THRRUN == 1) {
 
 		# If we are not the master server sleep and check again.
 		if (pandora_is_master($pa_config) == 0) {
@@ -4617,6 +4619,8 @@ sub pandora_process_policy_queue ($) {
 		
 		enterprise_hook('pandora_finish_queue_operation', [$dbh, $operation->{'id'}]);
 	}
+
+	db_disconnect($dbh);
 }
 
 ##########################################################################

pandora_server/lib/PandoraFMS/Tools.pm

@@ -77,6 +77,7 @@ our @EXPORT = qw(
 	MODULE_WARNING
 	MODULE_UNKNOWN
 	MODULE_NOTINIT
+	$THRRUN
 	api_call_url
 	cron_get_closest_in_range
 	cron_next_execution
@@ -115,6 +116,9 @@ our @EXPORT = qw(
 	valid_regex
 	set_file_permissions
 	uri_encode
+	check_server_threads
+	start_server_thread
+	stop_server_threads
 );
 
 # ID of the different servers
@@ -307,6 +311,12 @@ while (my ($ent, $chr) = each(%ENT2CHR)) {
 	$CHR2ENT{$chr} = "&" . $ent . ";";
 }
 
+# Threads started by the Pandora FMS Server.
+my @ServerThreads;
+
+# Keep threads running.
+our $THRRUN :shared = 1;
+
 ###############################################################################
 # Sets user:group owner for the given file
 ###############################################################################
@@ -1740,6 +1750,48 @@ sub api_call_url {
 	return undef;
 }
 
+################################################################################
+# Start a server thread and keep track of it.
+################################################################################
+sub start_server_thread {
+	my ($fn, $args) = @_;
+
+	# Signal the threads to run.
+	$THRRUN = 1;
+
+	my $thr = threads->create($fn, @{$args});
+	push(@ServerThreads, $thr);
+}
+
+################################################################################
+# Check the status of server threads. Returns 1 if all all running, 0 otherwise.
+################################################################################
+sub check_server_threads {
+	my ($fn, $args) = @_;
+
+	foreach my $thr (@ServerThreads) {
+		return 0 unless $thr->is_running();
+	}
+
+	return 1;
+}
+
+################################################################################
+# Stop all server threads.
+################################################################################
+sub stop_server_threads {
+	my ($fn, $args) = @_;
+
+	# Signal the threads to exits.
+	$THRRUN = 0;
+
+	foreach my $thr (@ServerThreads) {
+			$thr->detach();
+	}
+
+	@ServerThreads = ();
+}
+
 # End of function declaration
 # End of defined Code