Faster XML processing in LIFO mode.

Ref pandora_enterprise#2253

pandora_server/lib/PandoraFMS/Core.pm

@@ -1596,6 +1596,31 @@ sub pandora_process_module ($$$$$$$$$;$) {
 	my $ff_start_utimestamp = $agent_status->{'ff_start_utimestamp'};
 	my $mark_for_update = 0;
 	
+	# tagente_estado.last_try defaults to NULL, should default to '1970-01-01 00:00:00'
+	$agent_status->{'last_try'} = '1970-01-01 00:00:00' unless defined ($agent_status->{'last_try'});
+	$agent_status->{'datos'} = "" unless defined($agent_status->{'datos'});
+	
+	# Do we have to save module data?
+	if ($agent_status->{'last_try'} !~ /(\d+)\-(\d+)\-(\d+) +(\d+):(\d+):(\d+)/) {
+		logger($pa_config, "Invalid last try timestamp '" . $agent_status->{'last_try'} . "' for agent '" . $agent->{'nombre'} . "' not found while processing module '" . $module->{'nombre'} . "'.", 3);
+		pandora_update_module_on_error ($pa_config, $module, $dbh);
+		return;
+	}
+	my $last_try = ($1 == 0) ? 0 : timelocal($6, $5, $4, $3, $2 - 1, $1 - 1900);
+	my $save = ($module->{'history_data'} == 1 && ($agent_status->{'datos'} ne $processed_data || $last_try < ($utimestamp - 86400))) ? 1 : 0;
+	
+	# Received stale data. Save module data if needed and return.
+	if ($pa_config->{'dataserver_lifo'} == 1 && $utimestamp <= $agent_status->{'utimestamp'}) {
+		logger($pa_config, "Received stale data from agent " . (defined ($agent) ? "'" . $agent->{'nombre'} . "'" : 'ID ' . $module->{'id_agente'}) . ".", 10);
+		
+		# Save module data. Async and log4x modules are not compressed.
+		if ($module_type =~ m/(async)|(log4x)/ || $save == 1) {
+			save_module_data ($data_object, $module, $module_type, $utimestamp, $dbh);
+		}
+
+		return;
+	}
+
 	# Get new status
 	my $new_status = get_module_status ($processed_data, $module, $module_type);
 	
@@ -1744,23 +1769,6 @@ sub pandora_process_module ($$$$$$$$$;$) {
 		$mark_for_update = 1;
 	}
 		
-	# tagente_estado.last_try defaults to NULL, should default to '1970-01-01 00:00:00'
-	$agent_status->{'last_try'} = '1970-01-01 00:00:00' unless defined ($agent_status->{'last_try'});
-	
-	# Do we have to save module data?
-	if ($agent_status->{'last_try'} !~ /(\d+)\-(\d+)\-(\d+) +(\d+):(\d+):(\d+)/) {
-		logger($pa_config, "Invalid last try timestamp '" . $agent_status->{'last_try'} . "' for agent '" . $agent->{'nombre'} . "' not found while processing module '" . $module->{'nombre'} . "'.", 3);
-		pandora_update_module_on_error ($pa_config, $module, $dbh);
-		return;
-	}
-	
-	my $last_try = ($1 == 0) ? 0 : timelocal($6, $5, $4, $3, $2 - 1, $1 - 1900);
-
-	if (!defined($agent_status->{'datos'})){
-		$agent_status->{'datos'} = "";
-	}
-
-	my $save = ($module->{'history_data'} == 1 && ($agent_status->{'datos'} ne $processed_data || $last_try < ($utimestamp - 86400))) ? 1 : 0;
 	
 	# Never update tagente_estado when processing out-of-order data.
 	if ($utimestamp >= $last_try) {

pandora_server/lib/PandoraFMS/DataServer.pm

@@ -149,14 +149,8 @@ sub data_producer ($) {
 		next if ($file !~ /^(.*)[\._]\d+\.data$/);
 		my $agent_name = $1;
 
-		$AgentSem->down ();
+		next if (agent_lock($pa_config, $agent_name) == 0);
-		if (defined ($Agents{$agent_name})) {
+			
-			$AgentSem->up ();
-			next;
-		}
-		$Agents{$agent_name} = 1;
-		$AgentSem->up ();
-
 		push (@tasks, $file);
 	}
 
@@ -181,9 +175,7 @@ sub data_consumer ($$) {
 
 	# Double check that the file exists
 	if (! -f $file_name) {
-		$AgentSem->down ();
+		agent_unlock($pa_config, $agent_name);
-		delete ($Agents{$agent_name});
-		$AgentSem->up ();
 		return;
 	}
 
@@ -213,9 +205,7 @@ sub data_consumer ($$) {
 
 		# Double check that the file exists
 		if (! -f $file_name) {
-			$AgentSem->down ();
+			agent_unlock($pa_config, $agent_name);
-			delete ($Agents{$agent_name});
-			$AgentSem->up ();
 			return;
 		}
 
@@ -231,17 +221,13 @@ sub data_consumer ($$) {
 		} else {
 			process_xml_data ($self->getConfig (), $file_name, $xml_data, $self->getServerID (), $self->getDBH ());
 		}
-		$AgentSem->down ();
+		agent_unlock($pa_config, $agent_name);
-		delete ($Agents{$agent_name});
-		$AgentSem->up ();
 		return;	
 	}
 
 	rename($file_name, $file_name . '_BADXML');
 	pandora_event ($pa_config, "Unable to process XML data file '$file_name': $xml_err", 0, 0, 0, 0, 0, 'error', 0, $dbh);
-	$AgentSem->down ();
+	agent_unlock($pa_config, $agent_name);
-	delete ($Agents{$agent_name});
-	$AgentSem->up ();
 }
 
 ###############################################################################
@@ -1059,5 +1045,37 @@ sub process_xml_matrix_network {
 	return;
 }
 
+##########################################################################
+# Get a lock on the given agent. Return 1 on success, 0 otherwise.
+##########################################################################
+sub agent_lock {
+	my ($pa_config, $agent_name) = @_;
+
+	return 1 if ($pa_config->{'dataserver_lifo'} == 1);
+
+	$AgentSem->down ();
+	if (defined ($Agents{$agent_name})) {
+		$AgentSem->up ();
+		return 0;
+	}
+	$Agents{$agent_name} = 1;
+	$AgentSem->up ();
+
+	return 1;
+}
+
+##########################################################################
+# Remove the lock on the given agent.
+##########################################################################
+sub agent_unlock {
+	my ($pa_config, $agent_name) = @_;
+
+	return if ($pa_config->{'dataserver_lifo'} == 1);
+
+	$AgentSem->down ();
+	delete ($Agents{$agent_name});
+	$AgentSem->up ();
+}
+
 1;
 __END__