From 14aa8083cbeb8e9dd007fb487ee9561e8d51cd11 Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Sun, 1 Oct 2023 14:48:13 -0600 Subject: [PATCH 01/11] Add logs module --- pandora_agents/pc/pandora_agent | 186 ++++++++++++++++++- pandora_agents/unix/Linux/pandora_agent.conf | 10 + pandora_agents/win32/bin/pandora_agent.conf | 9 + 3 files changed, 204 insertions(+), 1 deletion(-) diff --git a/pandora_agents/pc/pandora_agent b/pandora_agents/pc/pandora_agent index aa8208f5d0..798d0ba5af 100644 --- a/pandora_agents/pc/pandora_agent +++ b/pandora_agents/pc/pandora_agent @@ -394,6 +394,8 @@ sub parse_conf_modules($) { 'max_warning' => undef, 'disabled' => undef, 'min_ff_event' => undef, + 'filter' => undef, + 'log_file' => undef, 'save' => '', 'conditions' => [], 'cron' => '', @@ -411,7 +413,7 @@ sub parse_conf_modules($) { $module->{'description'} = $1; } elsif ($line =~ /^\s*module_type\s+(\S+)\s*$/) { $module->{'type'} = $1; - }elsif ($line =~ /^\s*module_precondition\s+(.*)$/) { + } elsif ($line =~ /^\s*module_precondition\s+(.*)$/) { my $action = $1; # Numeric comparison @@ -559,6 +561,16 @@ sub parse_conf_modules($) { # Min ff event } elsif ($line =~ /^\s*module_min_ff_event\s+(.*)\s*$/) { $module->{'min_ff_event'} = $1; + # Log module file + } elsif ($line =~ /^\s*module_logfile\s+(.*)\s*$/) { + $module->{'filter'} = $1; + # Log module filter + } elsif ($line =~ /^\s*module_filter\s+(.*)\s*$/) { + $module->{'log_file'} = $1; + # Log module function + } elsif ($line =~ /^\s*module_logger\s+(.*)\s*$/) { + $module->{'func'} = \&module_logger; + $module->{'params'} = $1; } } return; @@ -1814,6 +1826,178 @@ sub exec_plugin ($) { $ThreadSem->up () if (defined ($ThreadSem) && $Conf{'agent_threads'} > 1); } +################################################################################ +# Read the logs +################################################################################ +sub module_logger ($) { + + # Return: 0 If all was OK + # 1 If there is an error + my $status = grep_logs( + $module->{'name'}, + $module->{'log_file'}, + $module->{'filter'} + ) + + return ($status); +} + +sub grep_logs { + my ($str_name, $str_file, $str_regex) = @_; + if(!$str_name){ + log_message("module_logger", "Missing module name"); + return; + } + + if(!$str_file){ + log_message("module_logger", "Missing file name"); + return; + } + + if(!$str_regex){ + $str_regex = '\.\*' + } + + my $idx_dir = '/tmp/'; + my $idx_file = ''; + my $idx_pos = 0; + my $idx_ino = ''; + my $module_name = $str_name; + my $log_file = $str_file; + my $reg_exp = $str_regex; + + # Check that log file exists + if (! -e $log_file) { + log_message("module_logger", "File $log_file does not exist"); + return; + } + + # Create index file storage directory + if (! -d $idx_dir) { + if (!mkdir($idx_dir)){ + log_message("module_logger", "Error creating directory $idx_dir: " . $!); + return; + } + } + + # Create index file if it does not exist + my $idx_file = $idx_dir.$module_name."_".basename($log_file).".idx"; + if (! -e $idx_file) { + create_idx(); + } else { + return if load_idx() == 1; + + return if parse_log() == 1; + } + + # Start the function definition + sub create_idx { + my $first_line; + log_message("module_logger", "Creating index file $idx_file"); + if (!open(LOGFILE, $log_file)){ + log_message("module_logger", "Error opening file $log_file: ".$!); + return 1; + } + + # Go to EOF and save the position + seek(LOGFILE, 0, 2); + $idx_pos = tell(LOGFILE); + + close(LOGFILE); + + # Save the file inode number + $idx_ino = (stat($log_file))[1]; + + return 1 if save_idx() == 1; + + return 0; + } + + sub save_idx { + log_message("module_logger", "Saving index file $idx_file"); + + if (!open(IDXFILE, "> $idx_file")){ + log_message("module_logger", "Error opening file $idx_file: ". $!); + return 1; + } + + print (IDXFILE $idx_pos . " " . $idx_ino); + close(IDXFILE); + + return 0; + } + + sub load_idx { + my $line; + my $current_ino; + + log_message("module_logger", "Loading index file $idx_file"); + + if (!open(IDXFILE, $idx_file)){ + log_message("module_logger", "Error opening file $idx_file: " .$!); + return 1; + } + + # Read position and date + $line = ; + ($idx_pos, $idx_ino) = split(' ', $line); + + close(IDXFILE); + + # Reset the file index if the file has changed + $current_ino = (stat($log_file))[1]; + if ($current_ino != $idx_ino) { + log_message("module_logger", "File changed, resetting index"); + + $idx_pos = 0; + $idx_ino = $current_ino; + } + + return 0; + } + + sub parse_log { + my $line; + + log_message("module_logger", "Parsing log file $log_file"); + + # Open log file for reading + if (!open(LOGFILE, $log_file)){ + log_message("module_logger", "Error opening file $log_file: " . $!); + return 1; + } + + # Go to starting position. + seek(LOGFILE, $idx_pos, 0); + + print STDOUT "\n"; + print STDOUT "\n"; + print STDOUT "\n"; + print STDOUT "\n"; + + # Parse log file + while ($line = ) { + if ($line =~ m/$reg_exp/i) { + # Remove the trailing '\n' + chop($line); + + print STDOUT "\n"; + } + } + + print STDOUT "\n"; + print STDOUT "\n"; + + $idx_pos = tell(LOGFILE); + close(LOGFILE); + + # Save the index file + return 1 if save_idx() == 1; + + return 0; + } +} + ################################################################################ # TERM Handler ################################################################################ diff --git a/pandora_agents/unix/Linux/pandora_agent.conf b/pandora_agents/unix/Linux/pandora_agent.conf index 1b8131a42d..83966e6772 100644 --- a/pandora_agents/unix/Linux/pandora_agent.conf +++ b/pandora_agents/unix/Linux/pandora_agent.conf @@ -315,4 +315,14 @@ module_plugin autodiscover --default #module_begin #module_plugin /usr/share/pandora_agent/plugins/pandora_sca #module_absoluteinterval 7d +#module_end + +# Logs extaction plugin +#module_begin +#module_name Syslog +#module_type log +#module_logfile /var/log/messages +#module_logger syslog +# module_filter uses REGEXP, optional, if not defined, it takes all lines. +#module_filter \.\* #module_end \ No newline at end of file diff --git a/pandora_agents/win32/bin/pandora_agent.conf b/pandora_agents/win32/bin/pandora_agent.conf index d299325f3b..d19981d179 100644 --- a/pandora_agents/win32/bin/pandora_agent.conf +++ b/pandora_agents/win32/bin/pandora_agent.conf @@ -530,3 +530,12 @@ module_plugin "%PROGRAMFILES%\Pandora_Agent\util\autodiscover.exe" --default #module_absoluteinterval 7d #module_end +# Logs extaction plugin +#module_begin +#module_name Oracle_Server_log +#module_type log +#module_logfile c:\oracle\logs\oraserver.log +#module_logger syslog +#module_filter uses REGEXP, optional, if not defined, it takes all lines. +#module_filter \.\* +#module_end From 0177d1906d774a4b075d7530decc0f6a97f1bbb1 Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Mon, 2 Oct 2023 18:13:05 -0600 Subject: [PATCH 02/11] Changes on log result --- pandora_agents/pc/pandora_agent | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pandora_agents/pc/pandora_agent b/pandora_agents/pc/pandora_agent index 798d0ba5af..58b415ba0e 100644 --- a/pandora_agents/pc/pandora_agent +++ b/pandora_agents/pc/pandora_agent @@ -1883,11 +1883,14 @@ sub grep_logs { # Create index file if it does not exist my $idx_file = $idx_dir.$module_name."_".basename($log_file).".idx"; if (! -e $idx_file) { - create_idx(); - } else { + return if create_idx() == 1; + } else{ return if load_idx() == 1; + my $result = parse_log(); - return if parse_log() == 1; + return if $result == 1; + + return $result; } # Start the function definition @@ -1920,7 +1923,7 @@ sub grep_logs { log_message("module_logger", "Error opening file $idx_file: ". $!); return 1; } - + print (IDXFILE $idx_pos . " " . $idx_ino); close(IDXFILE); @@ -1957,6 +1960,7 @@ sub grep_logs { } sub parse_log { + my $result = ""; my $line; log_message("module_logger", "Parsing log file $log_file"); @@ -1970,10 +1974,10 @@ sub grep_logs { # Go to starting position. seek(LOGFILE, $idx_pos, 0); - print STDOUT "\n"; - print STDOUT "\n"; - print STDOUT "\n"; - print STDOUT "\n"; + $result = $result . "\n"; + $result = $result . "\n"; + $result = $result . "\n"; + $result = $result . "\n"; # Parse log file while ($line = ) { @@ -1981,20 +1985,21 @@ sub grep_logs { # Remove the trailing '\n' chop($line); - print STDOUT "\n"; + $result = $result . "\n"; } } - print STDOUT "\n"; - print STDOUT "\n"; + $result = $result . "\n"; + $result = $result . "\n"; $idx_pos = tell(LOGFILE); close(LOGFILE); + print($result); # Save the index file return 1 if save_idx() == 1; - return 0; + return $result; } } From 3bd306fcf1cb48f095569ddab79fa17b026be896 Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Wed, 4 Oct 2023 14:00:29 -0600 Subject: [PATCH 03/11] Modify log module --- pandora_agents/pc/pandora_agent | 199 +----------------- pandora_agents/unix/Linux/pandora_agent.conf | 15 +- pandora_agents/unix/pandora_agent | 205 +++++++++++++++++++ 3 files changed, 214 insertions(+), 205 deletions(-) diff --git a/pandora_agents/pc/pandora_agent b/pandora_agents/pc/pandora_agent index 58b415ba0e..67e603b225 100644 --- a/pandora_agents/pc/pandora_agent +++ b/pandora_agents/pc/pandora_agent @@ -54,7 +54,6 @@ if (!$@) { use constant AGENT_VERSION => '4.0.1'; use constant AGENT_BUILD => '111213'; - # Commands to retrieve total memory information in kB use constant TOTALMEMORY_CMDS => { linux => 'cat /proc/meminfo | grep MemTotal: | awk \'{ print $2 }\'', @@ -117,7 +116,6 @@ my $ConfDir = ''; # Pandora FMS agent configuration file my $ConfFile = 'pandora_agent.conf'; - # Broker agent configuration files my @BrokerPid; @@ -264,7 +262,6 @@ sub valid_regexp ($) { sub rmrf { my $path = shift; local *DIR; - if (-d $path) { opendir (DIR, $path) || return; while (defined (my $file_name = readdir(DIR))) { @@ -348,7 +345,6 @@ sub log_message ($$;$) { } } } - ################################################################################ # Add the given directory to the PATH. ################################################################################ @@ -394,8 +390,6 @@ sub parse_conf_modules($) { 'max_warning' => undef, 'disabled' => undef, 'min_ff_event' => undef, - 'filter' => undef, - 'log_file' => undef, 'save' => '', 'conditions' => [], 'cron' => '', @@ -413,7 +407,7 @@ sub parse_conf_modules($) { $module->{'description'} = $1; } elsif ($line =~ /^\s*module_type\s+(\S+)\s*$/) { $module->{'type'} = $1; - } elsif ($line =~ /^\s*module_precondition\s+(.*)$/) { + }elsif ($line =~ /^\s*module_precondition\s+(.*)$/) { my $action = $1; # Numeric comparison @@ -561,16 +555,6 @@ sub parse_conf_modules($) { # Min ff event } elsif ($line =~ /^\s*module_min_ff_event\s+(.*)\s*$/) { $module->{'min_ff_event'} = $1; - # Log module file - } elsif ($line =~ /^\s*module_logfile\s+(.*)\s*$/) { - $module->{'filter'} = $1; - # Log module filter - } elsif ($line =~ /^\s*module_filter\s+(.*)\s*$/) { - $module->{'log_file'} = $1; - # Log module function - } elsif ($line =~ /^\s*module_logger\s+(.*)\s*$/) { - $module->{'func'} = \&module_logger; - $module->{'params'} = $1; } } return; @@ -594,7 +578,6 @@ sub write_broker_conf($){ } while (my $line = ){ - # Skip broker definitions if ($line =~ m/^\s*broker_agent/) { next; @@ -1822,187 +1805,9 @@ sub exec_plugin ($) { $Sem->down () if (defined ($Sem)); $Xml .= $output; $Sem->up () if (defined ($Sem)); - $ThreadSem->up () if (defined ($ThreadSem) && $Conf{'agent_threads'} > 1); } -################################################################################ -# Read the logs -################################################################################ -sub module_logger ($) { - - # Return: 0 If all was OK - # 1 If there is an error - my $status = grep_logs( - $module->{'name'}, - $module->{'log_file'}, - $module->{'filter'} - ) - - return ($status); -} - -sub grep_logs { - my ($str_name, $str_file, $str_regex) = @_; - if(!$str_name){ - log_message("module_logger", "Missing module name"); - return; - } - - if(!$str_file){ - log_message("module_logger", "Missing file name"); - return; - } - - if(!$str_regex){ - $str_regex = '\.\*' - } - - my $idx_dir = '/tmp/'; - my $idx_file = ''; - my $idx_pos = 0; - my $idx_ino = ''; - my $module_name = $str_name; - my $log_file = $str_file; - my $reg_exp = $str_regex; - - # Check that log file exists - if (! -e $log_file) { - log_message("module_logger", "File $log_file does not exist"); - return; - } - - # Create index file storage directory - if (! -d $idx_dir) { - if (!mkdir($idx_dir)){ - log_message("module_logger", "Error creating directory $idx_dir: " . $!); - return; - } - } - - # Create index file if it does not exist - my $idx_file = $idx_dir.$module_name."_".basename($log_file).".idx"; - if (! -e $idx_file) { - return if create_idx() == 1; - } else{ - return if load_idx() == 1; - my $result = parse_log(); - - return if $result == 1; - - return $result; - } - - # Start the function definition - sub create_idx { - my $first_line; - log_message("module_logger", "Creating index file $idx_file"); - if (!open(LOGFILE, $log_file)){ - log_message("module_logger", "Error opening file $log_file: ".$!); - return 1; - } - - # Go to EOF and save the position - seek(LOGFILE, 0, 2); - $idx_pos = tell(LOGFILE); - - close(LOGFILE); - - # Save the file inode number - $idx_ino = (stat($log_file))[1]; - - return 1 if save_idx() == 1; - - return 0; - } - - sub save_idx { - log_message("module_logger", "Saving index file $idx_file"); - - if (!open(IDXFILE, "> $idx_file")){ - log_message("module_logger", "Error opening file $idx_file: ". $!); - return 1; - } - - print (IDXFILE $idx_pos . " " . $idx_ino); - close(IDXFILE); - - return 0; - } - - sub load_idx { - my $line; - my $current_ino; - - log_message("module_logger", "Loading index file $idx_file"); - - if (!open(IDXFILE, $idx_file)){ - log_message("module_logger", "Error opening file $idx_file: " .$!); - return 1; - } - - # Read position and date - $line = ; - ($idx_pos, $idx_ino) = split(' ', $line); - - close(IDXFILE); - - # Reset the file index if the file has changed - $current_ino = (stat($log_file))[1]; - if ($current_ino != $idx_ino) { - log_message("module_logger", "File changed, resetting index"); - - $idx_pos = 0; - $idx_ino = $current_ino; - } - - return 0; - } - - sub parse_log { - my $result = ""; - my $line; - - log_message("module_logger", "Parsing log file $log_file"); - - # Open log file for reading - if (!open(LOGFILE, $log_file)){ - log_message("module_logger", "Error opening file $log_file: " . $!); - return 1; - } - - # Go to starting position. - seek(LOGFILE, $idx_pos, 0); - - $result = $result . "\n"; - $result = $result . "\n"; - $result = $result . "\n"; - $result = $result . "\n"; - - # Parse log file - while ($line = ) { - if ($line =~ m/$reg_exp/i) { - # Remove the trailing '\n' - chop($line); - - $result = $result . "\n"; - } - } - - $result = $result . "\n"; - $result = $result . "\n"; - - $idx_pos = tell(LOGFILE); - close(LOGFILE); - - print($result); - # Save the index file - return 1 if save_idx() == 1; - - return $result; - } -} - ################################################################################ # TERM Handler ################################################################################ @@ -2476,4 +2281,4 @@ This is released under the GNU Lesser General Public License. Copyright (c) 2005-2023 Pandora FMS -=cut +=cut \ No newline at end of file diff --git a/pandora_agents/unix/Linux/pandora_agent.conf b/pandora_agents/unix/Linux/pandora_agent.conf index 83966e6772..4659fa9f78 100644 --- a/pandora_agents/unix/Linux/pandora_agent.conf +++ b/pandora_agents/unix/Linux/pandora_agent.conf @@ -317,12 +317,11 @@ module_plugin autodiscover --default #module_absoluteinterval 7d #module_end -# Logs extaction plugin -#module_begin -#module_name Syslog -#module_type log -#module_logfile /var/log/messages -#module_logger syslog -# module_filter uses REGEXP, optional, if not defined, it takes all lines. -#module_filter \.\* +# Logs extaction +#module_begin +#module_name Syslog +#module_description Logs extaction module +#module_type log +#module_regexp /var/log/logfile.log +#module_pattern .* #module_end \ No newline at end of file diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index 919bbbc64f..1988db1bae 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -1562,6 +1562,9 @@ sub parse_conf_modules($) { } elsif ($line =~ /^\s*module_occupiedpercentdisk\s+(.*)$/) { $module->{'func'} = \&module_occupiedpercentdisk; $module->{'params'} = $1; + }elsif ($line =~ /^\s*module_regexp\s+(.*)$/) { + $module->{'func'} = \&module_logger; + $module->{'params'} = $1; } elsif ($line =~ /^\s*module_max\s+(.*)\s*$/) { $module->{'max'} = $1; } elsif ($line =~ /^\s*module_min\s+(.*)\s*$/) { @@ -1817,7 +1820,11 @@ sub parse_conf_modules($) { # Macros } elsif ($line =~ /^\s*module_macro(\S+)\s+(.*)\s*$/) { $module->{'macros'}{$1} = $2; + # Regexp } + elsif ($line =~ /^\s*module_pattern(\S+)\s+(.*)\s*$/) { + $module->{'filter'} = $1; + } } return; } @@ -3860,6 +3867,203 @@ sub module_plugin ($) { return ($output); } +################################################################################ +# Read the logs +################################################################################ +sub module_logger ($) { + my $module = shift; + + my $status = grep_logs( + $module->{'name'}, + $module->{'params'}, + $module->{'filter'} + ); + + print($status); + + return ($status); +} + +sub grep_logs { + my ($str_name, $str_file, $str_regex) = @_; + + if(!$str_name){ + log_message("module_logger", "Missing module name"); + return; + } + + if(!$str_file){ + log_message("module_logger", "Missing file name"); + return; + } + + if(!$str_regex){ + $str_regex = '.*'; + } + + my $idx_dir = '/tmp/'; + my $idx_file = ''; + my $idx_pos = 0; + my $idx_ino = ''; + my $module_name = $str_name; + my $log_file = $str_file; + my $reg_exp = $str_regex; + + # Check that log file exists + if (! -e $log_file) { + log_message("module_logger", "File $log_file does not exist"); + return; + } + + # Create index file storage directory + if (! -d $idx_dir) { + if (!mkdir($idx_dir)){ + log_message("module_logger", "Error creating directory $idx_dir: " . $!); + return; + } + } + + # Create index file if it does not exist + $idx_file = $idx_dir.$module_name."_".basename($log_file).".idx"; + if (! -e $idx_file) { + return if create_idx(\$idx_pos, \$idx_ino, \$idx_file, \$log_file) == 1; + return "" + } else{ + + return if load_idx(\$idx_pos, \$idx_ino, \$idx_file) == 1; + my $result = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp); + + if (looks_like_number($result)) { + return if $result == 1; + } + + return $result; + } + + # Start the function definition + sub create_idx { + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref) = @_; + my $first_line; + + + + log_message("module_logger", "Creating index file $$idx_file_ref"); + + if (!open(LOGFILE, $$log_file_ref)){ + log_message("module_logger", "Error opening file $$log_file_ref: ".$!); + return 1; + } + + # Go to EOF and save the position + seek(LOGFILE, 0, 2); + $$idx_pos_ref = tell(LOGFILE); + + ################### + + close(LOGFILE); + + # Save the file inode number + $$idx_ino_ref = (stat($$log_file_ref))[1]; + + return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref) == 1; + + return 0; + } + + sub save_idx { + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref) = @_; + + log_message("module_logger", "Saving index file $$idx_file_ref"); + + if (!open(IDXFILE, "> $$idx_file_ref")){ + log_message("module_logger", "Error opening file $$idx_file_ref: ". $!); + return 1; + } + + print (IDXFILE $$idx_pos_ref . " " . $$idx_ino_ref); + close(IDXFILE); + + return 0; + } + + sub load_idx { + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref) = @_; + my $line; + my $current_ino; + + log_message("module_logger", "Loading index file $$idx_file_ref"); + + if (!open(IDXFILE, $$idx_file_ref)){ + log_message("module_logger", "Error opening file $$idx_file_ref: " .$!); + return 1; + } + + # Read position and date + $line = ; + ($$idx_pos_ref, $$idx_ino_ref) = split(' ', $line); + + close(IDXFILE); + + print($$idx_pos_ref); + print($$idx_pos_ref); + + # Reset the file index if the file has changed + $current_ino = (stat($$idx_file_ref))[1]; + + if ($current_ino != $$idx_ino_ref) { + log_message("module_logger", "File changed, resetting index"); + + $$idx_pos_ref = 0; + $$idx_ino_ref = $current_ino; + } + + return 0; + } + + sub parse_log { + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref, $module_name_ref, $reg_exp_ref) = @_; + my $result = ""; + my $line; + + log_message("module_logger", "Parsing log file $$log_file_ref"); + + # Open log file for reading + if (!open(LOGFILE, $$log_file_ref)){ + log_message("module_logger", "Error opening file $$log_file_ref: " . $!); + return 1; + } + + # Go to starting position. + seek(LOGFILE, $$idx_pos_ref, 0); + + $result = $result . "\n"; + $result = $result . "\n"; + $result = $result . "\n"; + $result = $result . "\n"; + + # Parse log file + while ($line = ) { + if ($line =~ m/$$reg_exp_ref/i) { + # Remove the trailing '\n' + chop($line); + + $result = $result . "\n"; + } + } + + $result = $result . "\n"; + $result = $result . "\n"; + + $$idx_pos_ref = tell(LOGFILE); + close(LOGFILE); + + # Save the index file + return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref) == 1; + + return $result; + } +} + ################################################################################ # TERM Handler ################################################################################ @@ -4030,6 +4234,7 @@ sub init_module ($) { $module->{'module_ff_interval'} = undef; $module->{'macros'} = {}; $module->{'alert_template'} = undef; + $module->{'filter'} = undef; } ################################################################################ From 0ab1570b7d382e3add01bb8d248b76512047286b Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Wed, 4 Oct 2023 14:07:31 -0600 Subject: [PATCH 04/11] Modify conf --- pandora_agents/win32/bin/pandora_agent.conf | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pandora_agents/win32/bin/pandora_agent.conf b/pandora_agents/win32/bin/pandora_agent.conf index d19981d179..4325eaa6ce 100644 --- a/pandora_agents/win32/bin/pandora_agent.conf +++ b/pandora_agents/win32/bin/pandora_agent.conf @@ -530,12 +530,11 @@ module_plugin "%PROGRAMFILES%\Pandora_Agent\util\autodiscover.exe" --default #module_absoluteinterval 7d #module_end -# Logs extaction plugin -#module_begin -#module_name Oracle_Server_log -#module_type log -#module_logfile c:\oracle\logs\oraserver.log -#module_logger syslog -#module_filter uses REGEXP, optional, if not defined, it takes all lines. -#module_filter \.\* -#module_end +# Logs extaction +#module_begin +#module_name X_Server_log +#module_description Logs extaction module +#module_type log +#module_regexp C:\server\logs\xserver.log +#module_pattern .* +#module_end \ No newline at end of file From a5b6b3e0f32585d2b076cd0160ffe78544fe480b Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Wed, 4 Oct 2023 14:10:50 -0600 Subject: [PATCH 05/11] Modify conf files --- pandora_agents/unix/Linux/pandora_agent.conf | 2 +- pandora_agents/win32/bin/pandora_agent.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandora_agents/unix/Linux/pandora_agent.conf b/pandora_agents/unix/Linux/pandora_agent.conf index 4659fa9f78..10705b3e7e 100644 --- a/pandora_agents/unix/Linux/pandora_agent.conf +++ b/pandora_agents/unix/Linux/pandora_agent.conf @@ -317,7 +317,7 @@ module_plugin autodiscover --default #module_absoluteinterval 7d #module_end -# Logs extaction +# Logs extraction #module_begin #module_name Syslog #module_description Logs extaction module diff --git a/pandora_agents/win32/bin/pandora_agent.conf b/pandora_agents/win32/bin/pandora_agent.conf index 4325eaa6ce..0438696681 100644 --- a/pandora_agents/win32/bin/pandora_agent.conf +++ b/pandora_agents/win32/bin/pandora_agent.conf @@ -530,11 +530,11 @@ module_plugin "%PROGRAMFILES%\Pandora_Agent\util\autodiscover.exe" --default #module_absoluteinterval 7d #module_end -# Logs extaction +# Logs extraction #module_begin #module_name X_Server_log #module_description Logs extaction module #module_type log #module_regexp C:\server\logs\xserver.log #module_pattern .* -#module_end \ No newline at end of file +#module_end From ff81e3a68cf20c45d7d7aa29ff3e26c7fe06e09d Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Wed, 4 Oct 2023 14:16:30 -0600 Subject: [PATCH 06/11] Remove extra prints --- pandora_agents/unix/pandora_agent | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index 1988db1bae..6310c3466b 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -3879,8 +3879,6 @@ sub module_logger ($) { $module->{'filter'} ); - print($status); - return ($status); } @@ -4003,9 +4001,6 @@ sub grep_logs { ($$idx_pos_ref, $$idx_ino_ref) = split(' ', $line); close(IDXFILE); - - print($$idx_pos_ref); - print($$idx_pos_ref); # Reset the file index if the file has changed $current_ino = (stat($$idx_file_ref))[1]; From ea8fe5744adf5af051a748ce0f11a57dbff4c4ac Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Fri, 6 Oct 2023 12:15:24 -0600 Subject: [PATCH 07/11] Fix Linux log module --- pandora_agents/unix/pandora_agent | 115 ++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index 6310c3466b..e6f0d97521 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -3879,9 +3879,39 @@ sub module_logger ($) { $module->{'filter'} ); - return ($status); + return; } +my $encode_sub = defined(&MIME::Base64::encode_base64) ? \&MIME::Base64::encode_base64 : sub { + my ($str, $endl) = @_; + + my @ALPHABET = ('A'..'Z', 'a'..'z', 0..9, '+', '/'); + my $str_len = length($str); + my $str_base64 = ''; + + for (my $i = 0; $i < $str_len; $i += 3) { + my $chunk = substr($str, $i, 3); + my $chunk_len = length($chunk); + + my $num = 0; + $num |= ord(substr($chunk, 0, 1)) << 16 if ($chunk_len >= 1); + $num |= ord(substr($chunk, 1, 1)) << 8 if ($chunk_len >= 2); + $num |= ord(substr($chunk, 2, 1)) if ($chunk_len == 3); + + my $enc_1 = ($num & 0xfc0000) >> 18; + my $enc_2 = ($num & 0x03f000) >> 12; + my $enc_3 = ($num & 0x000fc0) >> 6; + my $enc_4 = ($num & 0x00003f); + + $str_base64 .= $ALPHABET[$enc_1]; + $str_base64 .= $ALPHABET[$enc_2]; + $str_base64 .= $chunk_len >= 2 ? $ALPHABET[$enc_3] : '='; + $str_base64 .= $chunk_len == 3 ? $ALPHABET[$enc_4] : '='; + } + + return $str_base64; +}; + sub grep_logs { my ($str_name, $str_file, $str_regex) = @_; @@ -3902,6 +3932,7 @@ sub grep_logs { my $idx_dir = '/tmp/'; my $idx_file = ''; my $idx_pos = 0; + my $idx_size = 0; my $idx_ino = ''; my $module_name = $str_name; my $log_file = $str_file; @@ -3924,27 +3955,24 @@ sub grep_logs { # Create index file if it does not exist $idx_file = $idx_dir.$module_name."_".basename($log_file).".idx"; if (! -e $idx_file) { - return if create_idx(\$idx_pos, \$idx_ino, \$idx_file, \$log_file) == 1; - return "" + return if create_idx(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$idx_size) == 1; + return } else{ - return if load_idx(\$idx_pos, \$idx_ino, \$idx_file) == 1; - my $result = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp); + return if load_idx(\$idx_pos, \$idx_ino, \$idx_file, \$idx_size) == 1; + my @data = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp, \$idx_size); - if (looks_like_number($result)) { - return if $result == 1; - } + print_log (@data); - return $result; + return; } # Start the function definition + sub create_idx { - my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref) = @_; + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref, $idx_size_ref) = @_; my $first_line; - - log_message("module_logger", "Creating index file $$idx_file_ref"); if (!open(LOGFILE, $$log_file_ref)){ @@ -3956,20 +3984,18 @@ sub grep_logs { seek(LOGFILE, 0, 2); $$idx_pos_ref = tell(LOGFILE); - ################### - close(LOGFILE); # Save the file inode number $$idx_ino_ref = (stat($$log_file_ref))[1]; - return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref) == 1; + return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $idx_size_ref) == 1; return 0; } sub save_idx { - my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref) = @_; + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $idx_size_ref) = @_; log_message("module_logger", "Saving index file $$idx_file_ref"); @@ -3978,16 +4004,17 @@ sub grep_logs { return 1; } - print (IDXFILE $$idx_pos_ref . " " . $$idx_ino_ref); + print (IDXFILE $$idx_pos_ref . " " . $$idx_ino_ref . " " . $$idx_size_ref); close(IDXFILE); return 0; } sub load_idx { - my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref) = @_; + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $idx_size_ref) = @_; my $line; my $current_ino; + my $current_size; log_message("module_logger", "Loading index file $$idx_file_ref"); @@ -3998,26 +4025,27 @@ sub grep_logs { # Read position and date $line = ; - ($$idx_pos_ref, $$idx_ino_ref) = split(' ', $line); + ($$idx_pos_ref, $$idx_ino_ref, $$idx_size_ref) = split(' ', $line); close(IDXFILE); # Reset the file index if the file has changed $current_ino = (stat($$idx_file_ref))[1]; - - if ($current_ino != $$idx_ino_ref) { + $current_size = -s "$$idx_file_ref"; + if ($current_ino != $$idx_ino_ref || $current_size < $$idx_size_ref) { log_message("module_logger", "File changed, resetting index"); $$idx_pos_ref = 0; $$idx_ino_ref = $current_ino; } + $$idx_size_ref = $current_size; return 0; } sub parse_log { - my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref, $module_name_ref, $reg_exp_ref) = @_; - my $result = ""; + my ($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $log_file_ref, $module_name_ref, $reg_exp_ref, $idx_size_ref) = @_; + my $line; log_message("module_logger", "Parsing log file $$log_file_ref"); @@ -4031,32 +4059,43 @@ sub grep_logs { # Go to starting position. seek(LOGFILE, $$idx_pos_ref, 0); - $result = $result . "\n"; - $result = $result . "\n"; - $result = $result . "\n"; - $result = $result . "\n"; - # Parse log file - while ($line = ) { + my @data; + while ($line = ) { if ($line =~ m/$$reg_exp_ref/i) { - # Remove the trailing '\n' - chop($line); - - $result = $result . "\n"; + push (@data, $line); } } - $result = $result . "\n"; - $result = $result . "\n"; - $$idx_pos_ref = tell(LOGFILE); close(LOGFILE); # Save the index file - return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref) == 1; + return 1 if save_idx($idx_pos_ref, $idx_ino_ref, $idx_file_ref, $idx_size_ref) == 1; - return $result; + return @data; } + + sub print_log (@) { + my (@data, $module_name) = @_; + + # No data + if ($#data < 0) { + return; + } + + # Log module + my $output = "\n"; + $output .= "\n"; + $output .= "base64\n"; + $output .= ""; + $output .= "\n"; + + print stdout $output; + } + } ################################################################################ From 8742bfb0cb5f7f22e1b55215231039d3df3e6320 Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Sat, 7 Oct 2023 18:58:51 -0600 Subject: [PATCH 08/11] Changes on function response --- pandora_agents/unix/pandora_agent | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index e6f0d97521..acd8883682 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -39,6 +39,14 @@ BEGIN { use File::Copy; use Scalar::Util qw(looks_like_number); + use File::Basename; + + BEGIN { + eval { + require MIME::Base64; + }; + } + BEGIN { push @INC, '/usr/lib/perl5'; } ################################################################################ @@ -3670,6 +3678,11 @@ sub write_module_xml ($@) { return; } + if ($module->{'func'} == \&module_logger) { + $Xml .= $data[0]; + return + } + # Critical section $Sem->down () if (defined ($Sem)); @@ -3872,14 +3885,14 @@ sub module_plugin ($) { ################################################################################ sub module_logger ($) { my $module = shift; - + my $status = grep_logs( $module->{'name'}, $module->{'params'}, $module->{'filter'} ); - return; + return $status; } my $encode_sub = defined(&MIME::Base64::encode_base64) ? \&MIME::Base64::encode_base64 : sub { @@ -3962,9 +3975,9 @@ sub grep_logs { return if load_idx(\$idx_pos, \$idx_ino, \$idx_file, \$idx_size) == 1; my @data = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp, \$idx_size); - print_log (@data); - - return; + my $output = create_log(@data); + + return $output; } # Start the function definition @@ -4076,7 +4089,7 @@ sub grep_logs { return @data; } - sub print_log (@) { + sub create_log (@) { my (@data, $module_name) = @_; # No data @@ -4090,10 +4103,10 @@ sub grep_logs { $output .= "base64\n"; $output .= ""; + $output .= "]]>\n"; $output .= "\n"; - print stdout $output; + return $output; } } From 13eb1e92f738803c9735789114cc03e2c4b5a2bb Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Mon, 23 Oct 2023 08:59:06 -0500 Subject: [PATCH 09/11] Fix: Add module_name in output log --- pandora_agents/unix/pandora_agent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index acd8883682..a8fb9a51c9 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -3975,7 +3975,7 @@ sub grep_logs { return if load_idx(\$idx_pos, \$idx_ino, \$idx_file, \$idx_size) == 1; my @data = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp, \$idx_size); - my $output = create_log(@data); + my $output = create_log(@data, $module_name); return $output; } From e81aadc812c1e6c052e352a8d7be91077d1d713c Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Mon, 23 Oct 2023 11:47:58 -0600 Subject: [PATCH 10/11] Fix on module name --- pandora_agents/unix/Linux/pandora_agent.conf | 2 +- pandora_agents/unix/pandora_agent | 8 +++++--- pandora_agents/win32/bin/pandora_agent.conf | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pandora_agents/unix/Linux/pandora_agent.conf b/pandora_agents/unix/Linux/pandora_agent.conf index 10705b3e7e..1a6eb562ee 100644 --- a/pandora_agents/unix/Linux/pandora_agent.conf +++ b/pandora_agents/unix/Linux/pandora_agent.conf @@ -320,7 +320,7 @@ module_plugin autodiscover --default # Logs extraction #module_begin #module_name Syslog -#module_description Logs extaction module +#module_description Logs extraction module #module_type log #module_regexp /var/log/logfile.log #module_pattern .* diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index a8fb9a51c9..886d0cdaa0 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -3975,7 +3975,7 @@ sub grep_logs { return if load_idx(\$idx_pos, \$idx_ino, \$idx_file, \$idx_size) == 1; my @data = parse_log(\$idx_pos, \$idx_ino, \$idx_file, \$log_file, \$module_name, \$reg_exp, \$idx_size); - my $output = create_log(@data, $module_name); + my $output = create_log($module_name, @data); return $output; } @@ -4089,8 +4089,10 @@ sub grep_logs { return @data; } - sub create_log (@) { - my (@data, $module_name) = @_; + sub create_log($$){ + my ($module_name, @data) = @_; + log_message("Debug", $module_name); + log_message("Debug", join('', @data)); # No data if ($#data < 0) { diff --git a/pandora_agents/win32/bin/pandora_agent.conf b/pandora_agents/win32/bin/pandora_agent.conf index 0438696681..949d630851 100644 --- a/pandora_agents/win32/bin/pandora_agent.conf +++ b/pandora_agents/win32/bin/pandora_agent.conf @@ -533,7 +533,7 @@ module_plugin "%PROGRAMFILES%\Pandora_Agent\util\autodiscover.exe" --default # Logs extraction #module_begin #module_name X_Server_log -#module_description Logs extaction module +#module_description Logs extraction module #module_type log #module_regexp C:\server\logs\xserver.log #module_pattern .* From b7cad880283d053516f57f137263c84915203bec Mon Sep 17 00:00:00 2001 From: "felix.suarez" Date: Mon, 23 Oct 2023 11:53:38 -0600 Subject: [PATCH 11/11] Remove debug logs --- pandora_agents/unix/pandora_agent | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandora_agents/unix/pandora_agent b/pandora_agents/unix/pandora_agent index 886d0cdaa0..041ee0e5da 100755 --- a/pandora_agents/unix/pandora_agent +++ b/pandora_agents/unix/pandora_agent @@ -4091,8 +4091,6 @@ sub grep_logs { sub create_log($$){ my ($module_name, @data) = @_; - log_message("Debug", $module_name); - log_message("Debug", join('', @data)); # No data if ($#data < 0) {