tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ent-2253-5693-nueva-funcionalidad-para-el-procesado-mas-rapido-de-xml-de-un-mismo-agente-descartando-alertas-y-eventos' into 'develop' 

Faster XML processing in LIFO mode.

See merge request artica/pandorafms!2573
This commit is contained in:
Daniel Rodriguez 2019-07-23 13:00:13 +02:00
parent 776fcc3f81 13de5dbd51
commit ddc4e2f843
2 changed files with 63 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
pandora_server/lib/PandoraFMS/Core.pm
View File

@ -1596,6 +1596,31 @@ sub pandora_process_module ($$$$$$$$$;$) {
my $ff_start_utimestamp = $agent_status->{'ff_start_utimestamp'};
my $mark_for_update = 0;
# tagente_estado.last_try defaults to NULL, should default to '1970-01-01 00:00:00'
$agent_status->{'last_try'} = '1970-01-01 00:00:00' unless defined ($agent_status->{'last_try'});
$agent_status->{'datos'} = "" unless defined($agent_status->{'datos'});
# Do we have to save module data?
if ($agent_status->{'last_try'} !~ /(\d+)\-(\d+)\-(\d+) +(\d+):(\d+):(\d+)/) {
logger($pa_config, "Invalid last try timestamp '" . $agent_status->{'last_try'} . "' for agent '" . $agent->{'nombre'} . "' not found while processing module '" . $module->{'nombre'} . "'.", 3);
pandora_update_module_on_error ($pa_config, $module, $dbh);
return;
}
my $last_try = ($1 == 0) ? 0 : timelocal($6, $5, $4, $3, $2 - 1, $1 - 1900);
my $save = ($module->{'history_data'} == 1 && ($agent_status->{'datos'} ne $processed_data || $last_try < ($utimestamp - 86400))) ? 1 : 0;
# Received stale data. Save module data if needed and return.
if ($pa_config->{'dataserver_lifo'} == 1 && $utimestamp <= $agent_status->{'utimestamp'}) {
logger($pa_config, "Received stale data from agent " . (defined ($agent) ? "'" . $agent->{'nombre'} . "'" : 'ID ' . $module->{'id_agente'}) . ".", 10);
# Save module data. Async and log4x modules are not compressed.
if ($module_type =~ m/(async)|(log4x)/ || $save == 1) {
save_module_data ($data_object, $module, $module_type, $utimestamp, $dbh);
}
return;
}
# Get new status
my $new_status = get_module_status ($processed_data, $module, $module_type);
@ -1744,23 +1769,6 @@ sub pandora_process_module ($$$$$$$$$;$) {
$mark_for_update = 1;
}
# tagente_estado.last_try defaults to NULL, should default to '1970-01-01 00:00:00'
$agent_status->{'last_try'} = '1970-01-01 00:00:00' unless defined ($agent_status->{'last_try'});
# Do we have to save module data?
if ($agent_status->{'last_try'} !~ /(\d+)\-(\d+)\-(\d+) +(\d+):(\d+):(\d+)/) {
logger($pa_config, "Invalid last try timestamp '" . $agent_status->{'last_try'} . "' for agent '" . $agent->{'nombre'} . "' not found while processing module '" . $module->{'nombre'} . "'.", 3);
pandora_update_module_on_error ($pa_config, $module, $dbh);
return;
}
my $last_try = ($1 == 0) ? 0 : timelocal($6, $5, $4, $3, $2 - 1, $1 - 1900);
if (!defined($agent_status->{'datos'})){
$agent_status->{'datos'} = "";
}
my $save = ($module->{'history_data'} == 1 && ($agent_status->{'datos'} ne $processed_data || $last_try < ($utimestamp - 86400))) ? 1 : 0;
# Never update tagente_estado when processing out-of-order data.
if ($utimestamp >= $last_try) {

58
pandora_server/lib/PandoraFMS/DataServer.pm
View File

@ -149,14 +149,8 @@ sub data_producer ($) {
next if ($file !~ /^(.*)[\._]\d+\.data$/);
my $agent_name = $1;
$AgentSem->down ();
if (defined ($Agents{$agent_name})) {
$AgentSem->up ();
next;
}
$Agents{$agent_name} = 1;
$AgentSem->up ();
next if (agent_lock($pa_config, $agent_name) == 0);
push (@tasks, $file);
}
@ -181,9 +175,7 @@ sub data_consumer ($$) {
# Double check that the file exists
if (! -f $file_name) {
$AgentSem->down ();
delete ($Agents{$agent_name});
$AgentSem->up ();
agent_unlock($pa_config, $agent_name);
return;
}
@ -213,9 +205,7 @@ sub data_consumer ($$) {
# Double check that the file exists
if (! -f $file_name) {
$AgentSem->down ();
delete ($Agents{$agent_name});
$AgentSem->up ();
agent_unlock($pa_config, $agent_name);
return;
}
@ -231,17 +221,13 @@ sub data_consumer ($$) {
} else {
process_xml_data ($self->getConfig (), $file_name, $xml_data, $self->getServerID (), $self->getDBH ());
}
$AgentSem->down ();
delete ($Agents{$agent_name});
$AgentSem->up ();
agent_unlock($pa_config, $agent_name);
return;
}
rename($file_name, $file_name . '_BADXML');
pandora_event ($pa_config, "Unable to process XML data file '$file_name': $xml_err", 0, 0, 0, 0, 0, 'error', 0, $dbh);
$AgentSem->down ();
delete ($Agents{$agent_name});
$AgentSem->up ();
agent_unlock($pa_config, $agent_name);
}
###############################################################################
@ -1059,5 +1045,37 @@ sub process_xml_matrix_network {
return;
}
##########################################################################
# Get a lock on the given agent. Return 1 on success, 0 otherwise.
##########################################################################
sub agent_lock {
my ($pa_config, $agent_name) = @_;
return 1 if ($pa_config->{'dataserver_lifo'} == 1);
$AgentSem->down ();
if (defined ($Agents{$agent_name})) {
$AgentSem->up ();
return 0;
}
$Agents{$agent_name} = 1;
$AgentSem->up ();
return 1;
}
##########################################################################
# Remove the lock on the given agent.
##########################################################################
sub agent_unlock {
my ($pa_config, $agent_name) = @_;
return if ($pa_config->{'dataserver_lifo'} == 1);
$AgentSem->down ();
delete ($Agents{$agent_name});
$AgentSem->up ();
}
1;
__END__