tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Unify file transfer, secondary mode and XML buffer criteria. 

Ref pandora_enterprise#5360
This commit is contained in:
Ramon Novoa 2020-01-29 13:00:17 +01:00
parent 0cf42f2d74
commit dac5b6b6b3
5 changed files with 156 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
pandora_agents/unix/pandora_agent
View File

@ -183,6 +183,7 @@ my %DefaultConf = (
'secondary_server_pwd' => '', 'secondary_server_pwd' => '',
'secondary_server_ssl' => '0', 'secondary_server_ssl' => '0',
'secondary_server_opts' => '', 'secondary_server_opts' => '',
'secondary_temporal' => '/var/spool/pandora',
'autotime' => 0, 'autotime' => 0,
'temporal_min_size' => 1, 'temporal_min_size' => 1,
'timezone_offset' => 0, 'timezone_offset' => 0,
@ -1058,6 +1059,16 @@ sub read_config (;$) {
$Conf{'secondary_server_opts'} = '-c ' . $Conf{'secondary_server_opts'} if ($Conf{'secondary_server_ssl'} eq '1'); $Conf{'secondary_server_opts'} = '-c ' . $Conf{'secondary_server_opts'} if ($Conf{'secondary_server_ssl'} eq '1');
} }
# Set up the primary and secondary temporary directories.
if ($Conf{'secondary_mode'} eq 'always') {
$Conf{'secondary_temporal'} = $Conf{'temporal'} . '/pandorafms.secondary';
if (! -d $Conf{'secondary_temporal'}) {
mkdir($Conf{'secondary_temporal'}) || die("Error creating the secondary temporary directory $!");
}
} elsif ($Conf{'secondary_temporal'} eq "on_error") {
$Conf{'secondary_temporal'} = $Conf{'temporal'};
}
} }
################################################################################# #################################################################################
@ -1077,7 +1088,7 @@ sub fix_directory ($) {
# Sends a file to the server. # Sends a file to the server.
################################################################################ ################################################################################
sub send_file { sub send_file {
my ($file, $secondary, $rc_primary, $flag_always, $relative) = @_; my ($file, $relative) = @_;
my $output; my $output;
my $pid = fork(); my $pid = fork();
@ -1131,108 +1142,82 @@ sub send_file {
waitpid ($pid, 0); waitpid ($pid, 0);
my $rc = $? >> 8; my $rc = $? >> 8;
if( ($Conf{'secondary_mode'} eq 'always') && ( !defined($flag_always) ) ){ return $rc
# Send the file to the secondary server }
return $rc unless ($Conf{'secondary_mode'} eq 'always');
if(defined ($secondary)){
if( ($rc != 0 && ($file =~ /\.data/)) ){
$rc_primary = 1;
}
swap_servers ();
$rc = send_file ($file, undef, $rc_primary, undef, $relative);
swap_servers ();
return $rc; ################################################################################
} # Send buffered XML files.
else{ ################################################################################
my $rc_secondary = 0; sub send_xml_file ($) {
if( ($rc != 0) && ($file =~ /\.data/)){ my ($file) = @_;
$rc_secondary = 1;
}
if ( $rc_secondary == 1 && defined($rc_primary) ){ my $rc = send_file($file);
return 1; if ($rc != 0 && $Conf{'secondary_mode'} eq "on_error") {
} swap_servers();
$rc = send_file($file);
if ( $rc_secondary == 1 ){ swap_servers();
if (! -d "$Conf{'temporal'}/secondary"){ }
mkdir "$Conf{'temporal'}/secondary"; elsif ($Conf{'secondary_mode'} eq "always") {
} swap_servers();
eval { my $rc_sec = send_file($file);
copy("$file", "$Conf{'temporal'}/secondary/"); swap_servers();
};
if ($@) {
# We shouldn't reach this point...
die ("Cannot write on $Conf{'temporal'}/secondary/");
}
return 0;
}
if ( defined($rc_primary) ){
if (! -d "$Conf{'temporal'}/primary"){
mkdir "$Conf{'temporal'}/primary";
}
eval {
copy("$file", "$Conf{'temporal'}/primary/");
};
if ($@) {
# We shouldn't reach this point...
die ("Cannot write on $Conf{'temporal'}/primary/");
}
return 0;
}
if ( $rc_secondary == 0 && !defined($rc_primary) ){ # Secondary buffer.
return 0; if ($rc_sec != 0 && $Conf{'xml_buffer'} == 1 && temporal_freedisk () > $Conf{'temporal_min_size'}) {
} copy($file, $Conf{'secondary_temporal'}) || die("Error copying file $file to " . $Conf{'secondary_temporal'} . ": $!");
} }
} }
elsif ( ($Conf{'secondary_mode'} eq 'always') && defined($flag_always) ){
return $rc;
}
else{
return $rc unless (defined ($secondary));
# Send the file to the secondary server # Primary buffer.
return $rc unless ($Conf{'secondary_mode'} eq 'always' || ($Conf{'secondary_mode'} eq 'on_error' && $rc != 0)); if ($rc == 0 || $Conf{'xml_buffer'} == 0 || temporal_freedisk () <= $Conf{'temporal_min_size'}) {
if ($Conf{'debug'} eq '1') {
swap_servers (); rename($file, $file . "sent");
$rc = send_file ($file, undef, undef, undef, $relative); } else {
swap_servers (); unlink ($file);
return $rc; }
} }
} }
################################################################################ ################################################################################
# Send buffered XML files. # Send buffered XML files.
################################################################################ ################################################################################
sub send_buffered_xml_files ($;$) { sub send_buffered_xml_files () {
my ($temporal_file, $flag_always) = @_; my $temp_fh;
# Read XML files from the temporal directory # Read XML files from the temporal directory
opendir(TEMPORAL, $temporal_file) or return; opendir($temp_fh, $Conf{'temporal'}) or return;
if (defined($flag_always) && ($flag_always == 2)){ while (my $xml_file = readdir($temp_fh)) {
swap_servers ();
}
while (my $xml_file = readdir(TEMPORAL)) {
# Skip non data files and symlinks # Skip non data files and symlinks
next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$temporal_file/$xml_file"); next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$Conf{'temporal'}/$xml_file");
my $rc = send_file ("$temporal_file/$xml_file", 1, undef, $flag_always); my $rc = send_file ("$Conf{'temporal'}/$xml_file");
if ($rc == 0) { if ($rc == 0) {
if ($Conf{'debug'} eq '1') { if ($Conf{'debug'} eq '1') {
rename "$temporal_file/$xml_file", "$temporal_file/$xml_file". "sent"; rename("$Conf{'temporal'}/$xml_file", "$Conf{'temporal'}/$xml_file". "sent");
} else { } else {
unlink ("$temporal_file/$xml_file"); unlink ("$Conf{'temporal'}/$xml_file");
} }
} } else {
# Do not get stuck trying to send buffered XML files to a secondary server.
elsif ($flag_always == 2) {
last; last;
} }
} }
if (defined($flag_always) && ($flag_always == 2)){ closedir($temp_fh);
swap_servers ();
# Read XML files from the secondary temporal directory
return unless ($Conf{'secondary_temporal'} ne "never");
opendir($temp_fh, $Conf{'secondary_temporal'}) or return;
swap_servers ();
while (my $xml_file = readdir($temp_fh)) {
# Skip non data files and symlinks
next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$Conf{'secondary_temporal'}/$xml_file");
my $rc = send_file ("$Conf{'secondary_temporal'}/$xml_file");
if ($rc == 0) {
unlink ("$Conf{'secondary_temporal'}/$xml_file") ;
} else {
last;
}
} }
swap_servers ();
closedir($temp_fh);
} }
################################################################################ ################################################################################
@ -1341,8 +1326,8 @@ sub check_remote_config () {
chown ($uid, $gid, "$Conf{'temporal'}/$RemoteMD5File"); chown ($uid, $gid, "$Conf{'temporal'}/$RemoteMD5File");
chown ($uid, $gid, "$Conf{'temporal'}/$RemoteConfFile"); chown ($uid, $gid, "$Conf{'temporal'}/$RemoteConfFile");
} }
send_file ("$Conf{'temporal'}/$RemoteConfFile", undef, undef, undef, $Conf{'server_path_conf'}); send_file ("$Conf{'temporal'}/$RemoteConfFile", $Conf{'server_path_conf'});
send_file ("$Conf{'temporal'}/$RemoteMD5File", undef, undef, undef, $Conf{'server_path_md5'}); send_file ("$Conf{'temporal'}/$RemoteMD5File", $Conf{'server_path_md5'});
unlink ("$Conf{'temporal'}/$RemoteConfFile"); unlink ("$Conf{'temporal'}/$RemoteConfFile");
unlink ("$Conf{'temporal'}/$RemoteMD5File"); unlink ("$Conf{'temporal'}/$RemoteMD5File");
return; return;
@ -3710,31 +3695,11 @@ while (1) {
} }
# Send the XML data file # Send the XML data file
my $rc = send_file ($temp_file, 1); send_xml_file ($temp_file);
if ($rc == 0 || $Conf{'xml_buffer'} == 0 || temporal_freedisk () < $Conf{'temporal_min_size'}) {
if ($Conf{'debug'} eq '1') {
rename $temp_file, $temp_file . "sent";
} else {
unlink ($temp_file);
}
}
# Send buffered XML data files # Send buffered XML data files
if ($Conf{'xml_buffer'} == 1) { if ($Conf{'xml_buffer'} == 1) {
if($Conf{'secondary_mode'} eq 'always'){ send_buffered_xml_files ();
$Conf{'__temporal_primary'} = "$Conf{'temporal'}/primary";
$Conf{'__temporal_secondary'} = "$Conf{'temporal'}/secondary";
if (-d "$Conf{'__temporal_primary'}"){
send_buffered_xml_files ($Conf{'__temporal_primary'}, 1);
}
if (-d "$Conf{'__temporal_secondary'}"){
send_buffered_xml_files ($Conf{'__temporal_secondary'}, 2);
}
send_buffered_xml_files ($Conf{'temporal'});
}
else{
send_buffered_xml_files ($Conf{'temporal'});
}
} }
} }

18
pandora_agents/win32/misc/pandora_file.cc
View File

@ -31,6 +31,24 @@
using namespace std; using namespace std;
/**
* Checks if a directory exists.
*
* @param dirpath Path of the directory to check.
*
* @retval True if the directory exists.
**/
bool
Pandora_File::dirExists (const string dirpath) {
struct stat info;
if (stat(dirpath.c_str(), &info) == 0 && (info.st_mode & S_IFDIR)) {
return true;
}
return false;
}
/** /**
* Checks if a file exists. * Checks if a file exists.
* *

1
pandora_agents/win32/misc/pandora_file.h
View File

@ -51,6 +51,7 @@ namespace Pandora_File {
class Delete_Error : Pandora_File::File_Exception { class Delete_Error : Pandora_File::File_Exception {
}; };
bool dirExists (const string dirpath);
bool fileExists (const string filename); bool fileExists (const string filename);
int readFile (const string filepath, string &result); int readFile (const string filepath, string &result);
int readBinFile (const string filepath, char **buffer); int readBinFile (const string filepath, char **buffer);

80
pandora_agents/win32/pandora_windows_service.cc
View File

@ -47,6 +47,7 @@
using namespace std; using namespace std;
using namespace Pandora; using namespace Pandora;
using namespace Pandora_File;
using namespace Pandora_Modules; using namespace Pandora_Modules;
using namespace Pandora_Strutils; using namespace Pandora_Strutils;
@ -251,6 +252,22 @@ Pandora_Windows_Service::pandora_init (bool reload_modules) {
} }
} }
/* Set up the secondary buffer. */
if (conf->getValue ("secondary_mode") == "always") {
string secondary_temporal = conf->getValue("temporal");
if (secondary_temporal[secondary_temporal.length () - 1] != '\\') {
secondary_temporal += "\\";
}
secondary_temporal += SECONDARY_DIR;
if (!dirExists(secondary_temporal) && mkdir (secondary_temporal.c_str()) != 0) {
pandoraLog ("Pandora_Windows_Service::pandora_init: Can not create directory %s", secondary_temporal.c_str());
}
conf->setValue("secondary_temporal", secondary_temporal);
}
else if (conf->getValue ("secondary_mode") == "on_error") {
conf->setValue("secondary_temporal", conf->getValue("temporal"));
}
// Set the intensive interval // Set the intensive interval
if (intensive_interval != "") { if (intensive_interval != "") {
try { try {
@ -980,7 +997,7 @@ Pandora_Windows_Service::copyFtpDataFile (string host,
} }
int int
Pandora_Windows_Service::copyDataFile (string filename) Pandora_Windows_Service::copyDataFile (string filename, bool secondary_buffer)
{ {
int rc = 0, timeout; int rc = 0, timeout;
unsigned char copy_to_secondary = 0; unsigned char copy_to_secondary = 0;
@ -1020,19 +1037,18 @@ Pandora_Windows_Service::copyDataFile (string filename)
if (rc == 0) { if (rc == 0) {
pandoraDebug ("Successfuly copied XML file to server."); pandoraDebug ("Successfuly copied XML file to server.");
} else if (conf->getValue ("secondary_mode") == "on_error") {
copy_to_secondary = 1;
} }
if (conf->getValue ("secondary_mode") == "always") { return rc;
copy_to_secondary = 1; }
}
int
Pandora_Windows_Service::copyToSecondary (string filename, bool secondary_buffer)
{
int rc = 0, timeout;
unsigned char copy_to_secondary = 0;
string mode, host, remote_path;
// Exit unless we have to send the file to a secondary server
if (copy_to_secondary == 0) {
return rc;
}
// Read secondary server configuration // Read secondary server configuration
mode = conf->getValue ("secondary_transfer_mode"); mode = conf->getValue ("secondary_transfer_mode");
host = conf->getValue ("secondary_server_ip"); host = conf->getValue ("secondary_server_ip");
@ -1042,6 +1058,12 @@ Pandora_Windows_Service::copyDataFile (string filename)
timeout = 30; timeout = 30;
} }
// Adjust the path for the secondary buffer.
if (secondary_buffer) {
filename.insert(0, "\\");
filename.insert(0, SECONDARY_DIR);
}
// Fix remote path // Fix remote path
if (mode != "local" && remote_path[remote_path.length () - 1] != '/') { if (mode != "local" && remote_path[remote_path.length () - 1] != '/') {
remote_path += "/"; remote_path += "/";
@ -1061,7 +1083,7 @@ Pandora_Windows_Service::copyDataFile (string filename)
} else { } else {
rc = PANDORA_EXCEPTION; rc = PANDORA_EXCEPTION;
pandoraLog ("Invalid transfer mode: %s." pandoraLog ("Invalid transfer mode: %s."
"Please recheck transfer_mode option " "Please recheck secondary_transfer_mode option "
"in configuration file."); "in configuration file.");
} }
@ -1671,10 +1693,11 @@ Pandora_Windows_Service::checkConfig (string file) {
int int
Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) { Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) {
int rc = 0, xml_buffer; int rc = 0, rc_sec = 0, xml_buffer;
string data_xml; string data_xml;
string xml_filename, random_integer; string xml_filename, random_integer;
string tmp_filename, tmp_filepath; string tmp_filename, tmp_filepath;
string secondary_filename, secondary_filepath;
string encoding; string encoding;
string ehorus_conf, eh_key; string ehorus_conf, eh_key;
static HANDLE mutex = 0; static HANDLE mutex = 0;
@ -1779,8 +1802,19 @@ Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) {
/* Allways reports to Data Server*/ /* Allways reports to Data Server*/
rc = this->copyDataFile (tmp_filename); rc = this->copyDataFile (tmp_filename);
if (rc != 0 && conf->getValue("secondary_mode") == "on_error") {
rc = this->copyToSecondary (tmp_filename, false);
} else if (conf->getValue("secondary_mode") == "always") {
rc_sec = this->copyToSecondary (tmp_filename, false);
/* Secondary buffer. */
if (rc_sec != 0 && xml_buffer == 1 && (GetDiskFreeSpaceEx (conf->getValue ("secondary_temporal").c_str (), &free_bytes, NULL, NULL) != 0 && free_bytes.QuadPart >= min_free_bytes)) {
secondary_filepath = conf->getValue ("secondary_temporal") + "\\" + tmp_filename;
CopyFile (tmp_filepath.c_str(), secondary_filepath.c_str(), false);
}
}
/* Delete the file if successfully copied, buffer disabled or not enough space available */ /* Primary buffer. Delete the file if successfully copied, buffer disabled or not enough space available. */
if (rc == 0 || xml_buffer == 0 || (GetDiskFreeSpaceEx (tmp_filepath.c_str (), &free_bytes, NULL, NULL) != 0 && free_bytes.QuadPart < min_free_bytes)) { if (rc == 0 || xml_buffer == 0 || (GetDiskFreeSpaceEx (tmp_filepath.c_str (), &free_bytes, NULL, NULL) != 0 && free_bytes.QuadPart < min_free_bytes)) {
/* Rename the file if debug mode is enabled*/ /* Rename the file if debug mode is enabled*/
if (getPandoraDebug ()) { if (getPandoraDebug ()) {
@ -1793,18 +1827,28 @@ Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) {
/* Send any buffered data files */ /* Send any buffered data files */
if (xml_buffer == 1) { if (xml_buffer == 1) {
this->sendBufferedXml (conf->getValue ("temporal")); this->sendBufferedXml (conf->getValue ("temporal"), &Pandora_Windows_Service::copyDataFile, false);
if (conf->getValue ("secondary_mode") == "always") {
this->sendBufferedXml (conf->getValue ("secondary_temporal"), &Pandora_Windows_Service::copyToSecondary, true);
} else {
this->sendBufferedXml (conf->getValue ("temporal"), &Pandora_Windows_Service::copyToSecondary, false);
}
} }
ReleaseMutex (mutex); ReleaseMutex (mutex);
} }
void void
Pandora_Windows_Service::sendBufferedXml (string path) { Pandora_Windows_Service::sendBufferedXml (string path, copy_func_p copy_func, bool secondary_buffer) {
string base_path = path, file_path; string base_path = path, file_path;
WIN32_FIND_DATA file_data; WIN32_FIND_DATA file_data;
HANDLE find; HANDLE find;
/* Nothing to do. */
if (path == "") {
return;
}
if (base_path[base_path.length () - 1] != '\\') { if (base_path[base_path.length () - 1] != '\\') {
base_path += "\\"; base_path += "\\";
} }
@ -1817,7 +1861,7 @@ Pandora_Windows_Service::sendBufferedXml (string path) {
} }
/* Send data files as long as there are no errors */ /* Send data files as long as there are no errors */
if (this->copyDataFile (file_data.cFileName) != 0) { if ((this->*copy_func) (file_data.cFileName, secondary_buffer) != 0) {
FindClose(find); FindClose(find);
return; return;
} }
@ -1832,7 +1876,7 @@ Pandora_Windows_Service::sendBufferedXml (string path) {
Pandora_File::removeFile (base_path + file_data.cFileName); Pandora_File::removeFile (base_path + file_data.cFileName);
while (FindNextFile(find, &file_data) != 0) { while (FindNextFile(find, &file_data) != 0) {
if (this->copyDataFile (file_data.cFileName) != 0) { if ((this->*copy_func) (file_data.cFileName, secondary_buffer) != 0) {
FindClose(find); FindClose(find);
return; return;
} }

7
pandora_agents/win32/pandora_windows_service.h
View File

@ -30,6 +30,7 @@
#define FTP_DEFAULT_PORT 21 #define FTP_DEFAULT_PORT 21
#define SSH_DEFAULT_PORT 22 #define SSH_DEFAULT_PORT 22
#define SECONDARY_DIR "secondary" /* Path of the secondary buffer relative to the primary buffer. */
using namespace std; using namespace std;
using namespace Pandora_Modules; using namespace Pandora_Modules;
@ -39,6 +40,7 @@ namespace Pandora {
* Class to implement the Pandora Windows service. * Class to implement the Pandora Windows service.
*/ */
class Pandora_Windows_Service : public Windows_Service { class Pandora_Windows_Service : public Windows_Service {
typedef int (Pandora::Pandora_Windows_Service::*copy_func_p)(string, bool);
private: private:
Pandora_Agent_Conf *conf; Pandora_Agent_Conf *conf;
Pandora_Module_List *modules; Pandora_Module_List *modules;
@ -54,7 +56,8 @@ namespace Pandora {
list<string> collection_disk; list<string> collection_disk;
string getXmlHeader (); string getXmlHeader ();
int copyDataFile (string filename); int copyDataFile (string filename, bool secondary_buffer = false);
int copyToSecondary (string filename, bool secondary_buffer = true);
string getValueFromCmdExec (string cmd_exec, int timeout); string getValueFromCmdExec (string cmd_exec, int timeout);
string getAgentNameFromCmdExec (string cmd_exec); string getAgentNameFromCmdExec (string cmd_exec);
string getCoordinatesFromCmdExec (string cmd_exec); string getCoordinatesFromCmdExec (string cmd_exec);
@ -115,7 +118,7 @@ namespace Pandora {
void start (); void start ();
int sendXml (Pandora_Module_List *modules); int sendXml (Pandora_Module_List *modules);
void sendBufferedXml (string path); void sendBufferedXml (string path, copy_func_p copy_func, bool secondary_buffer);
Pandora_Agent_Conf *getConf (); Pandora_Agent_Conf *getConf ();
string getEHKey (string ehorus_conf); string getEHKey (string ehorus_conf);
long getInterval (); long getInterval ();