From dac5b6b6b38af5fe8e4a20dcea671eafe43cb9e5 Mon Sep 17 00:00:00 2001 From: Ramon Novoa Date: Wed, 29 Jan 2020 13:00:17 +0100 Subject: [PATCH] Unify file transfer, secondary mode and XML buffer criteria. Ref pandora_enterprise#5360 --- pandora_agents/unix/pandora_agent | 175 +++++++----------- pandora_agents/win32/misc/pandora_file.cc | 18 ++ pandora_agents/win32/misc/pandora_file.h | 1 + .../win32/pandora_windows_service.cc | 80 ++++++-- .../win32/pandora_windows_service.h | 7 +- 5 files changed, 156 insertions(+), 125 deletions(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index 07e898f683..be3d152421 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -183,6 +183,7 @@ my %DefaultConf = ( 'secondary_server_pwd' => '', 'secondary_server_ssl' => '0', 'secondary_server_opts' => '', + 'secondary_temporal' => '/var/spool/pandora', 'autotime' => 0, 'temporal_min_size' => 1, 'timezone_offset' => 0, @@ -1058,6 +1059,16 @@ sub read_config (;$) { $Conf{'secondary_server_opts'} = '-c ' . $Conf{'secondary_server_opts'} if ($Conf{'secondary_server_ssl'} eq '1'); } + # Set up the primary and secondary temporary directories. + if ($Conf{'secondary_mode'} eq 'always') { + $Conf{'secondary_temporal'} = $Conf{'temporal'} . '/pandorafms.secondary'; + if (! -d $Conf{'secondary_temporal'}) { + mkdir($Conf{'secondary_temporal'}) || die("Error creating the secondary temporary directory $!"); + } + } elsif ($Conf{'secondary_temporal'} eq "on_error") { + $Conf{'secondary_temporal'} = $Conf{'temporal'}; + } + } ################################################################################# @@ -1077,7 +1088,7 @@ sub fix_directory ($) { # Sends a file to the server. ################################################################################ sub send_file { - my ($file, $secondary, $rc_primary, $flag_always, $relative) = @_; + my ($file, $relative) = @_; my $output; my $pid = fork(); @@ -1131,108 +1142,82 @@ sub send_file { waitpid ($pid, 0); my $rc = $? >> 8; - if( ($Conf{'secondary_mode'} eq 'always') && ( !defined($flag_always) ) ){ - # Send the file to the secondary server - return $rc unless ($Conf{'secondary_mode'} eq 'always'); - - if(defined ($secondary)){ - if( ($rc != 0 && ($file =~ /\.data/)) ){ - $rc_primary = 1; - } - swap_servers (); - $rc = send_file ($file, undef, $rc_primary, undef, $relative); - swap_servers (); + return $rc +} - return $rc; - } - else{ - my $rc_secondary = 0; - if( ($rc != 0) && ($file =~ /\.data/)){ - $rc_secondary = 1; - } +################################################################################ +# Send buffered XML files. +################################################################################ +sub send_xml_file ($) { + my ($file) = @_; - if ( $rc_secondary == 1 && defined($rc_primary) ){ - return 1; - } - - if ( $rc_secondary == 1 ){ - if (! -d "$Conf{'temporal'}/secondary"){ - mkdir "$Conf{'temporal'}/secondary"; - } - eval { - copy("$file", "$Conf{'temporal'}/secondary/"); - }; - if ($@) { - # We shouldn't reach this point... - die ("Cannot write on $Conf{'temporal'}/secondary/"); - } - return 0; - } - - if ( defined($rc_primary) ){ - if (! -d "$Conf{'temporal'}/primary"){ - mkdir "$Conf{'temporal'}/primary"; - } - eval { - copy("$file", "$Conf{'temporal'}/primary/"); - }; - if ($@) { - # We shouldn't reach this point... - die ("Cannot write on $Conf{'temporal'}/primary/"); - } - return 0; - } + my $rc = send_file($file); + if ($rc != 0 && $Conf{'secondary_mode'} eq "on_error") { + swap_servers(); + $rc = send_file($file); + swap_servers(); + } + elsif ($Conf{'secondary_mode'} eq "always") { + swap_servers(); + my $rc_sec = send_file($file); + swap_servers(); - if ( $rc_secondary == 0 && !defined($rc_primary) ){ - return 0; - } + # Secondary buffer. + if ($rc_sec != 0 && $Conf{'xml_buffer'} == 1 && temporal_freedisk () > $Conf{'temporal_min_size'}) { + copy($file, $Conf{'secondary_temporal'}) || die("Error copying file $file to " . $Conf{'secondary_temporal'} . ": $!"); } } - elsif ( ($Conf{'secondary_mode'} eq 'always') && defined($flag_always) ){ - return $rc; - } - else{ - return $rc unless (defined ($secondary)); - # Send the file to the secondary server - return $rc unless ($Conf{'secondary_mode'} eq 'always' || ($Conf{'secondary_mode'} eq 'on_error' && $rc != 0)); - - swap_servers (); - $rc = send_file ($file, undef, undef, undef, $relative); - swap_servers (); - return $rc; + # Primary buffer. + if ($rc == 0 || $Conf{'xml_buffer'} == 0 || temporal_freedisk () <= $Conf{'temporal_min_size'}) { + if ($Conf{'debug'} eq '1') { + rename($file, $file . "sent"); + } else { + unlink ($file); + } } } ################################################################################ # Send buffered XML files. ################################################################################ -sub send_buffered_xml_files ($;$) { - my ($temporal_file, $flag_always) = @_; +sub send_buffered_xml_files () { + my $temp_fh; + # Read XML files from the temporal directory - opendir(TEMPORAL, $temporal_file) or return; - if (defined($flag_always) && ($flag_always == 2)){ - swap_servers (); - } - while (my $xml_file = readdir(TEMPORAL)) { + opendir($temp_fh, $Conf{'temporal'}) or return; + while (my $xml_file = readdir($temp_fh)) { # Skip non data files and symlinks - next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$temporal_file/$xml_file"); - my $rc = send_file ("$temporal_file/$xml_file", 1, undef, $flag_always); + next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$Conf{'temporal'}/$xml_file"); + my $rc = send_file ("$Conf{'temporal'}/$xml_file"); if ($rc == 0) { if ($Conf{'debug'} eq '1') { - rename "$temporal_file/$xml_file", "$temporal_file/$xml_file". "sent"; + rename("$Conf{'temporal'}/$xml_file", "$Conf{'temporal'}/$xml_file". "sent"); } else { - unlink ("$temporal_file/$xml_file"); + unlink ("$Conf{'temporal'}/$xml_file"); } - } - # Do not get stuck trying to send buffered XML files to a secondary server. - elsif ($flag_always == 2) { + } else { last; } } - if (defined($flag_always) && ($flag_always == 2)){ - swap_servers (); + closedir($temp_fh); + + # Read XML files from the secondary temporal directory + return unless ($Conf{'secondary_temporal'} ne "never"); + opendir($temp_fh, $Conf{'secondary_temporal'}) or return; + swap_servers (); + while (my $xml_file = readdir($temp_fh)) { + # Skip non data files and symlinks + next if ($xml_file !~ m/^$Conf{'agent_name'}\.[0-9]+\.data$/ || -l "$Conf{'secondary_temporal'}/$xml_file"); + my $rc = send_file ("$Conf{'secondary_temporal'}/$xml_file"); + if ($rc == 0) { + unlink ("$Conf{'secondary_temporal'}/$xml_file") ; + } else { + last; + } } + swap_servers (); + closedir($temp_fh); } ################################################################################ @@ -1341,8 +1326,8 @@ sub check_remote_config () { chown ($uid, $gid, "$Conf{'temporal'}/$RemoteMD5File"); chown ($uid, $gid, "$Conf{'temporal'}/$RemoteConfFile"); } - send_file ("$Conf{'temporal'}/$RemoteConfFile", undef, undef, undef, $Conf{'server_path_conf'}); - send_file ("$Conf{'temporal'}/$RemoteMD5File", undef, undef, undef, $Conf{'server_path_md5'}); + send_file ("$Conf{'temporal'}/$RemoteConfFile", $Conf{'server_path_conf'}); + send_file ("$Conf{'temporal'}/$RemoteMD5File", $Conf{'server_path_md5'}); unlink ("$Conf{'temporal'}/$RemoteConfFile"); unlink ("$Conf{'temporal'}/$RemoteMD5File"); return; @@ -3710,31 +3695,11 @@ while (1) { } # Send the XML data file - my $rc = send_file ($temp_file, 1); - if ($rc == 0 || $Conf{'xml_buffer'} == 0 || temporal_freedisk () < $Conf{'temporal_min_size'}) { - if ($Conf{'debug'} eq '1') { - rename $temp_file, $temp_file . "sent"; - } else { - unlink ($temp_file); - } - } + send_xml_file ($temp_file); # Send buffered XML data files if ($Conf{'xml_buffer'} == 1) { - if($Conf{'secondary_mode'} eq 'always'){ - $Conf{'__temporal_primary'} = "$Conf{'temporal'}/primary"; - $Conf{'__temporal_secondary'} = "$Conf{'temporal'}/secondary"; - if (-d "$Conf{'__temporal_primary'}"){ - send_buffered_xml_files ($Conf{'__temporal_primary'}, 1); - } - if (-d "$Conf{'__temporal_secondary'}"){ - send_buffered_xml_files ($Conf{'__temporal_secondary'}, 2); - } - send_buffered_xml_files ($Conf{'temporal'}); - } - else{ - send_buffered_xml_files ($Conf{'temporal'}); - } + send_buffered_xml_files (); } } diff --git a/pandora_agents/win32/misc/pandora_file.cc b/pandora_agents/win32/misc/pandora_file.cc index dde0209d22..316ea2637f 100644 --- a/pandora_agents/win32/misc/pandora_file.cc +++ b/pandora_agents/win32/misc/pandora_file.cc @@ -31,6 +31,24 @@ using namespace std; +/** + * Checks if a directory exists. + * + * @param dirpath Path of the directory to check. + * + * @retval True if the directory exists. + **/ +bool +Pandora_File::dirExists (const string dirpath) { + struct stat info; + + if (stat(dirpath.c_str(), &info) == 0 && (info.st_mode & S_IFDIR)) { + return true; + } + + return false; +} + /** * Checks if a file exists. * diff --git a/pandora_agents/win32/misc/pandora_file.h b/pandora_agents/win32/misc/pandora_file.h index 8c3db124f9..9ec1153ad6 100644 --- a/pandora_agents/win32/misc/pandora_file.h +++ b/pandora_agents/win32/misc/pandora_file.h @@ -51,6 +51,7 @@ namespace Pandora_File { class Delete_Error : Pandora_File::File_Exception { }; + bool dirExists (const string dirpath); bool fileExists (const string filename); int readFile (const string filepath, string &result); int readBinFile (const string filepath, char **buffer); diff --git a/pandora_agents/win32/pandora_windows_service.cc b/pandora_agents/win32/pandora_windows_service.cc index 73d0a5d482..4f14f2b7c5 100644 --- a/pandora_agents/win32/pandora_windows_service.cc +++ b/pandora_agents/win32/pandora_windows_service.cc @@ -47,6 +47,7 @@ using namespace std; using namespace Pandora; +using namespace Pandora_File; using namespace Pandora_Modules; using namespace Pandora_Strutils; @@ -251,6 +252,22 @@ Pandora_Windows_Service::pandora_init (bool reload_modules) { } } + /* Set up the secondary buffer. */ + if (conf->getValue ("secondary_mode") == "always") { + string secondary_temporal = conf->getValue("temporal"); + if (secondary_temporal[secondary_temporal.length () - 1] != '\\') { + secondary_temporal += "\\"; + } + secondary_temporal += SECONDARY_DIR; + if (!dirExists(secondary_temporal) && mkdir (secondary_temporal.c_str()) != 0) { + pandoraLog ("Pandora_Windows_Service::pandora_init: Can not create directory %s", secondary_temporal.c_str()); + } + conf->setValue("secondary_temporal", secondary_temporal); + } + else if (conf->getValue ("secondary_mode") == "on_error") { + conf->setValue("secondary_temporal", conf->getValue("temporal")); + } + // Set the intensive interval if (intensive_interval != "") { try { @@ -980,7 +997,7 @@ Pandora_Windows_Service::copyFtpDataFile (string host, } int -Pandora_Windows_Service::copyDataFile (string filename) +Pandora_Windows_Service::copyDataFile (string filename, bool secondary_buffer) { int rc = 0, timeout; unsigned char copy_to_secondary = 0; @@ -1020,19 +1037,18 @@ Pandora_Windows_Service::copyDataFile (string filename) if (rc == 0) { pandoraDebug ("Successfuly copied XML file to server."); - } else if (conf->getValue ("secondary_mode") == "on_error") { - copy_to_secondary = 1; } - if (conf->getValue ("secondary_mode") == "always") { - copy_to_secondary = 1; - } + return rc; +} + +int +Pandora_Windows_Service::copyToSecondary (string filename, bool secondary_buffer) +{ + int rc = 0, timeout; + unsigned char copy_to_secondary = 0; + string mode, host, remote_path; - // Exit unless we have to send the file to a secondary server - if (copy_to_secondary == 0) { - return rc; - } - // Read secondary server configuration mode = conf->getValue ("secondary_transfer_mode"); host = conf->getValue ("secondary_server_ip"); @@ -1042,6 +1058,12 @@ Pandora_Windows_Service::copyDataFile (string filename) timeout = 30; } + // Adjust the path for the secondary buffer. + if (secondary_buffer) { + filename.insert(0, "\\"); + filename.insert(0, SECONDARY_DIR); + } + // Fix remote path if (mode != "local" && remote_path[remote_path.length () - 1] != '/') { remote_path += "/"; @@ -1061,7 +1083,7 @@ Pandora_Windows_Service::copyDataFile (string filename) } else { rc = PANDORA_EXCEPTION; pandoraLog ("Invalid transfer mode: %s." - "Please recheck transfer_mode option " + "Please recheck secondary_transfer_mode option " "in configuration file."); } @@ -1671,10 +1693,11 @@ Pandora_Windows_Service::checkConfig (string file) { int Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) { - int rc = 0, xml_buffer; + int rc = 0, rc_sec = 0, xml_buffer; string data_xml; string xml_filename, random_integer; string tmp_filename, tmp_filepath; + string secondary_filename, secondary_filepath; string encoding; string ehorus_conf, eh_key; static HANDLE mutex = 0; @@ -1779,8 +1802,19 @@ Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) { /* Allways reports to Data Server*/ rc = this->copyDataFile (tmp_filename); + if (rc != 0 && conf->getValue("secondary_mode") == "on_error") { + rc = this->copyToSecondary (tmp_filename, false); + } else if (conf->getValue("secondary_mode") == "always") { + rc_sec = this->copyToSecondary (tmp_filename, false); + + /* Secondary buffer. */ + if (rc_sec != 0 && xml_buffer == 1 && (GetDiskFreeSpaceEx (conf->getValue ("secondary_temporal").c_str (), &free_bytes, NULL, NULL) != 0 && free_bytes.QuadPart >= min_free_bytes)) { + secondary_filepath = conf->getValue ("secondary_temporal") + "\\" + tmp_filename; + CopyFile (tmp_filepath.c_str(), secondary_filepath.c_str(), false); + } + } - /* Delete the file if successfully copied, buffer disabled or not enough space available */ + /* Primary buffer. Delete the file if successfully copied, buffer disabled or not enough space available. */ if (rc == 0 || xml_buffer == 0 || (GetDiskFreeSpaceEx (tmp_filepath.c_str (), &free_bytes, NULL, NULL) != 0 && free_bytes.QuadPart < min_free_bytes)) { /* Rename the file if debug mode is enabled*/ if (getPandoraDebug ()) { @@ -1793,18 +1827,28 @@ Pandora_Windows_Service::sendXml (Pandora_Module_List *modules) { /* Send any buffered data files */ if (xml_buffer == 1) { - this->sendBufferedXml (conf->getValue ("temporal")); + this->sendBufferedXml (conf->getValue ("temporal"), &Pandora_Windows_Service::copyDataFile, false); + if (conf->getValue ("secondary_mode") == "always") { + this->sendBufferedXml (conf->getValue ("secondary_temporal"), &Pandora_Windows_Service::copyToSecondary, true); + } else { + this->sendBufferedXml (conf->getValue ("temporal"), &Pandora_Windows_Service::copyToSecondary, false); + } } ReleaseMutex (mutex); } void -Pandora_Windows_Service::sendBufferedXml (string path) { +Pandora_Windows_Service::sendBufferedXml (string path, copy_func_p copy_func, bool secondary_buffer) { string base_path = path, file_path; WIN32_FIND_DATA file_data; HANDLE find; + /* Nothing to do. */ + if (path == "") { + return; + } + if (base_path[base_path.length () - 1] != '\\') { base_path += "\\"; } @@ -1817,7 +1861,7 @@ Pandora_Windows_Service::sendBufferedXml (string path) { } /* Send data files as long as there are no errors */ - if (this->copyDataFile (file_data.cFileName) != 0) { + if ((this->*copy_func) (file_data.cFileName, secondary_buffer) != 0) { FindClose(find); return; } @@ -1832,7 +1876,7 @@ Pandora_Windows_Service::sendBufferedXml (string path) { Pandora_File::removeFile (base_path + file_data.cFileName); while (FindNextFile(find, &file_data) != 0) { - if (this->copyDataFile (file_data.cFileName) != 0) { + if ((this->*copy_func) (file_data.cFileName, secondary_buffer) != 0) { FindClose(find); return; } diff --git a/pandora_agents/win32/pandora_windows_service.h b/pandora_agents/win32/pandora_windows_service.h index 47f6271d22..3f9a89d685 100644 --- a/pandora_agents/win32/pandora_windows_service.h +++ b/pandora_agents/win32/pandora_windows_service.h @@ -30,6 +30,7 @@ #define FTP_DEFAULT_PORT 21 #define SSH_DEFAULT_PORT 22 +#define SECONDARY_DIR "secondary" /* Path of the secondary buffer relative to the primary buffer. */ using namespace std; using namespace Pandora_Modules; @@ -39,6 +40,7 @@ namespace Pandora { * Class to implement the Pandora Windows service. */ class Pandora_Windows_Service : public Windows_Service { + typedef int (Pandora::Pandora_Windows_Service::*copy_func_p)(string, bool); private: Pandora_Agent_Conf *conf; Pandora_Module_List *modules; @@ -54,7 +56,8 @@ namespace Pandora { list collection_disk; string getXmlHeader (); - int copyDataFile (string filename); + int copyDataFile (string filename, bool secondary_buffer = false); + int copyToSecondary (string filename, bool secondary_buffer = true); string getValueFromCmdExec (string cmd_exec, int timeout); string getAgentNameFromCmdExec (string cmd_exec); string getCoordinatesFromCmdExec (string cmd_exec); @@ -115,7 +118,7 @@ namespace Pandora { void start (); int sendXml (Pandora_Module_List *modules); - void sendBufferedXml (string path); + void sendBufferedXml (string path, copy_func_p copy_func, bool secondary_buffer); Pandora_Agent_Conf *getConf (); string getEHKey (string ehorus_conf); long getInterval ();