Sort queued files before applying the one XML per agent limit.

(cherry picked from commit e113c7e850)
This commit is contained in:
Ramon Novoa 2015-08-10 16:46:10 +02:00
parent 686f698fe8
commit 30ed3dbad5
1 changed files with 31 additions and 22 deletions

View File

@ -96,6 +96,7 @@ sub data_producer ($) {
my @tasks; my @tasks;
my @files; my @files;
my @sorted;
# Open the incoming directory # Open the incoming directory
opendir (DIR, $pa_config->{'incomingdir'}) opendir (DIR, $pa_config->{'incomingdir'})
@ -104,12 +105,37 @@ sub data_producer ($) {
# Do not read more than max_queue_files files # Do not read more than max_queue_files files
my $file_count = 0; my $file_count = 0;
while (my $file = readdir (DIR)) { while (my $file = readdir (DIR)) {
# Data files must have the extension .data # Data files must have the extension .data
next if ($file !~ /^.*[\._]\d+\.data$/);
# Do not queue more than max_queue_files files
if ($file_count >= $pa_config->{"max_queue_files"}) {
last;
}
push (@files, $file);
$file_count++;
}
closedir(DIR);
# Sort the queue
{
# Temporarily disable warnings (some files may have been deleted)
no warnings;
if ($pa_config->{'dataserver_lifo'} == 0) {
@sorted = sort { -M $pa_config->{'incomingdir'} . "/$b" <=> -M $pa_config->{'incomingdir'} . "/$a" } (@files);
} else {
@sorted = sort { -M $pa_config->{'incomingdir'} . "/$a" <=> -M $pa_config->{'incomingdir'} . "/$b" } (@files);
}
}
# Do not process more than one XML from the same agent at the same time
foreach my $file (@sorted) {
next if ($file !~ /^(.*)[\._]\d+\.data$/); next if ($file !~ /^(.*)[\._]\d+\.data$/);
my $agent_name = $1;
# Do not process more than one XML from the same agent at the same time
my $agent_name = $1;
$AgentSem->down (); $AgentSem->down ();
if (defined ($Agents{$agent_name})) { if (defined ($Agents{$agent_name})) {
$AgentSem->up (); $AgentSem->up ();
@ -118,24 +144,7 @@ sub data_producer ($) {
$Agents{$agent_name} = 1; $Agents{$agent_name} = 1;
$AgentSem->up (); $AgentSem->up ();
push (@files, $file); push (@tasks, $file);
$file_count++;
# Do not queue more than max_queue_files files
if ($file_count >= $pa_config->{"max_queue_files"}) {
last;
}
}
closedir(DIR);
# Temporarily disable warnings (some files may have been deleted)
{
no warnings;
if ($pa_config->{'dataserver_lifo'} == 0) {
@tasks = sort { -C $pa_config->{'incomingdir'} . "/$b" <=> -C $pa_config->{'incomingdir'} . "/$a" } (@files);
} else {
@tasks = sort { -C $pa_config->{'incomingdir'} . "/$a" <=> -C $pa_config->{'incomingdir'} . "/$b" } (@files);
}
} }
return @tasks; return @tasks;