Merge branch 'pandora_enterprise#2333-intracom-snmp-trap-buffer-subsystem' into 'develop'
Add support for trap data coming from XML files. See merge request artica/pandorafms!1667
This commit is contained in:
vgilc
commit
323599f50e
|
|
@ -0,0 +1,250 @@
|
|||
#!/usr/bin/perl
|
||||
###############################################################################
|
||||
#
|
||||
# Copyright (c) 2018 Artica Soluciones Tecnologicas S.L.
|
||||
#
|
||||
# grep_log Perl script to search log files for a matching pattern. The last
|
||||
# searched position is saved in an index file so that consecutive
|
||||
# runs do not return the same results. The log file inode number is
|
||||
# also saved to detect log rotation.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; version 2 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
###############################################################################
|
||||
use strict;
|
||||
use File::Basename;
|
||||
use IO::Compress::Zip qw(zip $ZipError);
|
||||
use MIME::Base64;
|
||||
|
||||
# Be verbose
|
||||
my $Verbose = 0;
|
||||
|
||||
# Index file storage directory, with a trailing '/'
|
||||
my $Idx_dir=($^O =~ /win/i)?'.\\':'/tmp/';
|
||||
|
||||
# Log file
|
||||
my $Log_file = '';
|
||||
|
||||
# Index file
|
||||
my $Idx_file = '';
|
||||
|
||||
# Log file position index
|
||||
my $Idx_pos = 0;
|
||||
|
||||
# Log file inode number
|
||||
my $Idx_ino = '';
|
||||
|
||||
# Log file size
|
||||
my $Idx_size = 0;
|
||||
|
||||
###############################################################################
|
||||
# SUB error_msg
|
||||
# Print an error message and exit.
|
||||
###############################################################################
|
||||
sub error_msg ($) {
|
||||
my $err_msg = $_[0];
|
||||
|
||||
if (! -z $err_msg) {
|
||||
print(STDERR "[error] $err_msg.\n");
|
||||
}
|
||||
|
||||
exit 1;
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB print_help
|
||||
# Print a help message.
|
||||
###############################################################################
|
||||
sub print_help () {
|
||||
print "Usage: $0 <snmptrapd log file>\n";
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB log_msg
|
||||
# Print a log message.
|
||||
###############################################################################
|
||||
sub log_msg ($) {
|
||||
my $log_msg = $_[0];
|
||||
|
||||
if (! -z $log_msg && $Verbose == 1) {
|
||||
print(STDOUT "[log] $log_msg.\n");
|
||||
}
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB load_idx
|
||||
# Load index file.
|
||||
###############################################################################
|
||||
sub load_idx () {
|
||||
my $line;
|
||||
my $current_ino;
|
||||
my $current_size;
|
||||
|
||||
log_msg("Loading index file $Idx_file");
|
||||
|
||||
open(IDXFILE, $Idx_file) || error_msg("Error opening file $Idx_file: " .
|
||||
$!);
|
||||
|
||||
# Read position and date
|
||||
$line = <IDXFILE>;
|
||||
($Idx_pos, $Idx_ino, $Idx_size) = split(' ', $line);
|
||||
|
||||
close(IDXFILE);
|
||||
|
||||
# Reset the file index if the file has changed
|
||||
$current_ino = (stat($Log_file))[1];
|
||||
$current_size = -s "$Log_file";
|
||||
if ($current_ino != $Idx_ino || $current_size < $Idx_size) {
|
||||
log_msg("File changed, resetting index");
|
||||
|
||||
$Idx_pos = 0;
|
||||
$Idx_ino = $current_ino;
|
||||
}
|
||||
$Idx_size = $current_size;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB save_idx
|
||||
# Save index file.
|
||||
###############################################################################
|
||||
sub save_idx () {
|
||||
|
||||
log_msg("Saving index file $Idx_file");
|
||||
|
||||
open(IDXFILE, "> $Idx_file") || error_msg("Error opening file $Idx_file: "
|
||||
. $!);
|
||||
print (IDXFILE $Idx_pos . " " . $Idx_ino . " " . $Idx_size);
|
||||
close(IDXFILE);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB create_idx
|
||||
# Create index file.
|
||||
###############################################################################
|
||||
sub create_idx () {
|
||||
my $first_line;
|
||||
|
||||
log_msg("Creating index file $Idx_file");
|
||||
|
||||
open(LOGFILE, $Log_file) || error_msg("Error opening file $Log_file: " .
|
||||
$!);
|
||||
|
||||
# Go to EOF and save the position
|
||||
seek(LOGFILE, 0, 2);
|
||||
$Idx_pos = tell(LOGFILE);
|
||||
|
||||
close(LOGFILE);
|
||||
|
||||
# Save the file inode number
|
||||
$Idx_ino = (stat($Log_file))[1];
|
||||
|
||||
# Save the index file
|
||||
save_idx();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB parse_log
|
||||
# Parse log file starting from position $Idx_pos.
|
||||
###############################################################################
|
||||
sub parse_log () {
|
||||
my $line;
|
||||
|
||||
log_msg("Parsing log file $Log_file");
|
||||
|
||||
# Open log file for reading
|
||||
open(LOGFILE, $Log_file) || error_msg("Error opening file $Log_file: " .
|
||||
$!);
|
||||
|
||||
# Go to starting position.
|
||||
seek(LOGFILE, $Idx_pos, 0);
|
||||
|
||||
# Parse log file
|
||||
my $data;
|
||||
|
||||
# Matched line id
|
||||
my $matched_line = 0;
|
||||
|
||||
$/ = undef;
|
||||
$data = <LOGFILE>;
|
||||
|
||||
$Idx_pos = tell(LOGFILE);
|
||||
close(LOGFILE);
|
||||
|
||||
# Save the index file
|
||||
save_idx();
|
||||
|
||||
return \$data;
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# SUB parse_log
|
||||
# Print log data to STDOUT.
|
||||
###############################################################################
|
||||
sub print_log ($) {
|
||||
my $data = shift;
|
||||
my $zdata;
|
||||
|
||||
return unless defined($data) and $$data ne '';
|
||||
|
||||
if (!zip($data => \$zdata)) {
|
||||
error_msg("Compression error: $ZipError");
|
||||
return;
|
||||
}
|
||||
|
||||
print STDOUT "<trap_data><![CDATA[" . encode_base64($zdata, '') . "]]></trap_data>\n";
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
###############################################################################
|
||||
## Main
|
||||
###############################################################################
|
||||
###############################################################################
|
||||
|
||||
# Check command line parameters
|
||||
if ($#ARGV < 0) {
|
||||
print_help();
|
||||
exit 1;
|
||||
}
|
||||
|
||||
$Log_file = $ARGV[0];
|
||||
# Create index file storage directory
|
||||
if ( ! -d $Idx_dir) {
|
||||
mkdir($Idx_dir) || error_msg("Error creating directory $Idx_dir: "
|
||||
. $!);
|
||||
}
|
||||
|
||||
# Check that log file exists
|
||||
if (! -e $Log_file) {
|
||||
error_msg("File $Log_file does not exist");
|
||||
}
|
||||
|
||||
# Create index file if it does not exist
|
||||
$Idx_file=$Idx_dir . basename($Log_file) . ".idx";
|
||||
if (! -e $Idx_file) {
|
||||
create_idx();
|
||||
exit 0;
|
||||
}
|
||||
|
||||
# Load index file
|
||||
load_idx();
|
||||
|
||||
# Parse log file
|
||||
my $data = parse_log();
|
||||
|
||||
# Print output to STDOUT
|
||||
print_log ($data);
|
||||
|
||||
exit 0;
|
||||
|
|
@ -485,8 +485,10 @@ sub pandora_load_config {
|
|||
$pa_config->{"provisioningserver"} = 1; # 7.0 720
|
||||
$pa_config->{"provisioningserver_threads"} = 1; # 7.0 720
|
||||
$pa_config->{"provisioning_cache_interval"} = 300; # 7.0 720
|
||||
|
||||
|
||||
$pa_config->{"autoconfigure_agents"} = 1; # 7.0 725
|
||||
|
||||
$pa_config->{'snmp_extlog'} = ""; # 7.0 726
|
||||
|
||||
# Check for UID0
|
||||
if ($pa_config->{"quiet"} != 0){
|
||||
|
|
@ -1118,6 +1120,9 @@ sub pandora_load_config {
|
|||
elsif ($parametro =~ m/^autoconfigure_agents\s+([0-1])/i){
|
||||
$pa_config->{'autoconfigure_agents'}= clean_blank($1);
|
||||
}
|
||||
elsif ($parametro =~ m/^snmp_extlog\s(.*)/i) {
|
||||
$pa_config->{'snmp_extlog'} = clean_blank($1);
|
||||
}
|
||||
} # end of loop for parameter #
|
||||
|
||||
# Set to RDBMS' standard port
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ use Time::Local;
|
|||
use XML::Parser::Expat;
|
||||
use XML::Simple;
|
||||
use POSIX qw(setsid strftime);
|
||||
use IO::Uncompress::Unzip;
|
||||
|
||||
# For Reverse Geocoding
|
||||
use LWP::Simple;
|
||||
|
|
@ -592,6 +593,9 @@ sub process_xml_data ($$$$$) {
|
|||
# Process log modules
|
||||
enterprise_hook('process_log_data', [$pa_config, $data, $server_id, $agent_name,
|
||||
$interval, $timestamp, $dbh]);
|
||||
|
||||
# Process snmptrapd modules
|
||||
enterprise_hook('process_snmptrap_data', [$pa_config, $data, $server_id, $dbh]);
|
||||
}
|
||||
|
||||
##########################################################################
|
||||
|
|
|
|||
|
|
@ -48,8 +48,9 @@ my $TaskSem :shared;
|
|||
# Trap statistics by agent
|
||||
my %AGENTS = ();
|
||||
|
||||
# Index file management
|
||||
my ($IDX_FILE, $LAST_LINE, $LAST_SIZE) = ();
|
||||
# Index and buffer management for trap log files
|
||||
my $SNMPTRAPD = { 'log_file' => '', 'fd' => undef, 'idx_file' => '', 'last_line' => 0, 'last_size' => 0, 'read_ahead_line' => '', 'read_ahead_pos' => 0 };
|
||||
my $DATASERVER = { 'log_file' => '', 'fd' => undef, 'idx_file' => '', 'last_line' => 0, 'last_size' => 0, 'read_ahead_line' => '', 'read_ahead_pos' => 0 };
|
||||
|
||||
########################################################################################
|
||||
# SNMP Server class constructor.
|
||||
|
|
@ -65,33 +66,27 @@ sub new ($$$) {
|
|||
}
|
||||
|
||||
# Wait for the SNMP log file to be available
|
||||
my $log_file = $config->{'snmp_logfile'};
|
||||
sleep ($config->{'server_threshold'}) if (! -e $log_file);
|
||||
if (!open (SNMPLOGFILE, $log_file)) {
|
||||
logger ($config, ' [E] Could not open the SNMP log file ' . $config->{'snmp_logfile'} . ".", 1);
|
||||
print_message ($config, ' [E] Could not open the SNMP log file ' . $config->{'snmp_logfile'} . ".", 1);
|
||||
$SNMPTRAPD->{'log_file'} = $config->{'snmp_logfile'};
|
||||
sleep ($config->{'server_threshold'}) if (! -e $SNMPTRAPD->{'log_file'});
|
||||
if (!open ($SNMPTRAPD->{'fd'}, $SNMPTRAPD->{'log_file'})) {
|
||||
logger ($config, ' [E] Could not open the SNMP log file ' . $SNMPTRAPD->{'log_file'} . ".", 1);
|
||||
print_message ($config, ' [E] Could not open the SNMP log file ' . $SNMPTRAPD->{'log_file'} . ".", 1);
|
||||
return 1;
|
||||
}
|
||||
init_log_file($config, $SNMPTRAPD);
|
||||
|
||||
# Process index file, if available
|
||||
($IDX_FILE, $LAST_LINE, $LAST_SIZE) = ($log_file . '.index', 0, 0);
|
||||
if (-e $IDX_FILE) {
|
||||
open (INDEXFILE, $IDX_FILE) or return;
|
||||
my $idx_data = <INDEXFILE>;
|
||||
close INDEXFILE;
|
||||
($LAST_LINE, $LAST_SIZE) = split(/\s+/, $idx_data);
|
||||
}
|
||||
my $log_size = (stat ($log_file))[7];
|
||||
|
||||
# New SNMP log file found
|
||||
if ($log_size < $LAST_SIZE) {
|
||||
unlink ($IDX_FILE);
|
||||
($LAST_LINE, $LAST_SIZE) = (0, 0);
|
||||
# Create the Data Server SNMP log file if it does not exist.
|
||||
if (defined($config->{'snmp_extlog'}) && $config->{'snmp_extlog'} ne '') {
|
||||
$DATASERVER->{'log_file'} = $config->{'snmp_extlog'};
|
||||
open(TMPFD, '>', $DATASERVER->{'log_file'}) && close(TMPFD) if (! -e $DATASERVER->{'log_file'});
|
||||
if (!open ($DATASERVER->{'fd'}, $DATASERVER->{'log_file'})) {
|
||||
logger ($config, ' [E] Could not open the Data Server SNMP log file ' . $DATASERVER->{'log_file'} . ".", 1);
|
||||
print_message ($config, ' [E] Could not open the Data Server SNMP log file ' . $DATASERVER->{'log_file'} . ".", 1);
|
||||
return 1;
|
||||
}
|
||||
init_log_file($config, $DATASERVER);
|
||||
}
|
||||
|
||||
# Skip already processed lines
|
||||
read_snmplogfile() for (1..$LAST_LINE);
|
||||
|
||||
# Initialize semaphores and queues
|
||||
@TaskQueue = ();
|
||||
%PendingTasks = ();
|
||||
|
|
@ -140,41 +135,45 @@ sub data_producer ($) {
|
|||
%AGENTS = ();
|
||||
}
|
||||
|
||||
while (my $line_with_pos = read_snmplogfile()) {
|
||||
my $line;
|
||||
|
||||
$LAST_LINE++;
|
||||
($LAST_SIZE, $line) = @$line_with_pos;
|
||||
|
||||
chomp ($line);
|
||||
|
||||
# Update index file
|
||||
open INDEXFILE, '>' . $IDX_FILE;
|
||||
print INDEXFILE $LAST_LINE . ' ' . $LAST_SIZE;
|
||||
close INDEXFILE;
|
||||
set_file_permissions($pa_config, $IDX_FILE, "0666");
|
||||
|
||||
# Skip lines other than SNMP Trap logs
|
||||
next unless ($line =~ m/^SNMPv[12]\[\*\*\]/);
|
||||
|
||||
# Storm protection.
|
||||
my ($ver, $date, $time, $source, $null) = split(/\[\*\*\]/, $line, 5);
|
||||
next unless defined ($source);
|
||||
if (! defined ($AGENTS{$source})) {
|
||||
$AGENTS{$source}{'count'} = 1;
|
||||
$AGENTS{$source}{'event'} = 0;
|
||||
} else {
|
||||
$AGENTS{$source}{'count'} += 1;
|
||||
}
|
||||
if ($pa_config->{'snmp_storm_protection'} > 0 && $AGENTS{$source}{'count'} > $pa_config->{'snmp_storm_protection'}) {
|
||||
if ($AGENTS{$source}{'event'} == 0) {
|
||||
pandora_event ($pa_config, "Too many traps coming from $source. Silenced for " . int ($pa_config->{"snmp_storm_timeout"} / 60) . " minutes.", 0, 0, 4, 0, 0, 'system', 0, $dbh);
|
||||
for my $fs (($SNMPTRAPD, $DATASERVER)) {
|
||||
next unless defined($fs->{'fd'});
|
||||
reset_if_truncated($pa_config, $fs);
|
||||
while (my $line_with_pos = read_snmplogfile($fs)) {
|
||||
my $line;
|
||||
|
||||
$fs->{'last_line'}++;
|
||||
($fs->{'last_size'}, $line) = @$line_with_pos;
|
||||
|
||||
chomp ($line);
|
||||
|
||||
# Update index file
|
||||
open(my $idxfd, '>' . $fs->{'idx_file'});
|
||||
print $idxfd $fs->{'last_line'} . ' ' . $fs->{'last_size'};
|
||||
close $idxfd;
|
||||
set_file_permissions($pa_config, $fs->{'idx_file'}, "0666");
|
||||
|
||||
# Skip lines other than SNMP Trap logs
|
||||
next unless ($line =~ m/^SNMPv[12]\[\*\*\]/);
|
||||
|
||||
# Storm protection.
|
||||
my ($ver, $date, $time, $source, $null) = split(/\[\*\*\]/, $line, 5);
|
||||
next unless defined ($source);
|
||||
if (! defined ($AGENTS{$source})) {
|
||||
$AGENTS{$source}{'count'} = 1;
|
||||
$AGENTS{$source}{'event'} = 0;
|
||||
} else {
|
||||
$AGENTS{$source}{'count'} += 1;
|
||||
}
|
||||
$AGENTS{$source}{'event'} = 1;
|
||||
next;
|
||||
if ($pa_config->{'snmp_storm_protection'} > 0 && $AGENTS{$source}{'count'} > $pa_config->{'snmp_storm_protection'}) {
|
||||
if ($AGENTS{$source}{'event'} == 0) {
|
||||
pandora_event ($pa_config, "Too many traps coming from $source. Silenced for " . int ($pa_config->{"snmp_storm_timeout"} / 60) . " minutes.", 0, 0, 4, 0, 0, 'system', 0, $dbh);
|
||||
}
|
||||
$AGENTS{$source}{'event'} = 1;
|
||||
next;
|
||||
}
|
||||
|
||||
push (@tasks, $line);
|
||||
}
|
||||
|
||||
push (@tasks, $line);
|
||||
}
|
||||
|
||||
return @tasks;
|
||||
|
|
@ -437,23 +436,21 @@ sub start_snmptrapd ($) {
|
|||
# Read SNMP Log file with buffering (to handle multi-line Traps).
|
||||
# Return reference of array (file-pos, line-data) if successful, undef othersise.
|
||||
###############################################################################
|
||||
my $read_ahead_line; # buffer to save fetched ahead line
|
||||
my $read_ahead_pos;
|
||||
|
||||
sub read_snmplogfile()
|
||||
{
|
||||
sub read_snmplogfile($) {
|
||||
my ($fs) = @_;
|
||||
my $line;
|
||||
my $pos;
|
||||
|
||||
if(defined($read_ahead_line)) {
|
||||
if(defined($fs->{'read_ahead_line'})) {
|
||||
# Restore saved line
|
||||
$line = $read_ahead_line;
|
||||
$pos = $read_ahead_pos;
|
||||
$line = $fs->{'read_ahead_line'};
|
||||
$pos = $fs->{'read_ahead_pos'};
|
||||
}
|
||||
else {
|
||||
# No saved line
|
||||
$line = <SNMPLOGFILE>;
|
||||
$pos = tell(SNMPLOGFILE);
|
||||
my $fd = $fs->{'fd'};
|
||||
$line = <$fd>;
|
||||
$pos = tell($fs->{'fd'});
|
||||
}
|
||||
|
||||
return undef if (! defined($line));
|
||||
|
|
@ -462,20 +459,21 @@ sub read_snmplogfile()
|
|||
|
||||
# More lines ?
|
||||
while(1) {
|
||||
while($read_ahead_line = <SNMPLOGFILE>) {
|
||||
my $fd = $fs->{'fd'};
|
||||
while($fs->{'read_ahead_line'} = <$fd>) {
|
||||
|
||||
# Get current file position
|
||||
$read_ahead_pos = tell(SNMPLOGFILE);
|
||||
$fs->{'read_ahead_pos'} = tell($fs->{'fd'});
|
||||
|
||||
# Get out of the loop if you find another Trap
|
||||
last if($read_ahead_line =~ /^SNMP/ );
|
||||
last if($fs->{'read_ahead_line'} =~ /^SNMP/ );
|
||||
|
||||
# $read_ahead_line looks continued line...
|
||||
# $fs->{'read_ahead_line'} looks continued line...
|
||||
|
||||
# Append to the line and correct the position
|
||||
chomp($line);
|
||||
$line .= "$read_ahead_line";
|
||||
$pos = $read_ahead_pos;
|
||||
$line .= "$fs->{'read_ahead_line'}";
|
||||
$pos = $fs->{'read_ahead_pos'};
|
||||
}
|
||||
|
||||
# if $line looks incomplete, try to get continued line
|
||||
|
|
@ -490,6 +488,49 @@ sub read_snmplogfile()
|
|||
return [$pos, $line];
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Initialize the fs structure for a trap log file.
|
||||
###############################################################################
|
||||
sub init_log_file($$$) {
|
||||
my ($config, $fs) = @_;
|
||||
|
||||
# Process index file, if available
|
||||
($fs->{'idx_file'}, $fs->{'last_line'}, $fs->{'last_size'}) = ($fs->{'log_file'} . '.index', 0, 0);
|
||||
if (-e $fs->{'idx_file'}) {
|
||||
open (my $idxfd, $fs->{'idx_file'}) or return;
|
||||
my $idx_data = <$idxfd>;
|
||||
close $idxfd;
|
||||
($fs->{'last_line'}, $fs->{'last_size'}) = split(/\s+/, $idx_data);
|
||||
}
|
||||
my $log_size = (stat ($fs->{'log_file'}))[7];
|
||||
|
||||
# New SNMP log file found
|
||||
if ($log_size < $fs->{'last_size'}) {
|
||||
unlink ($fs->{'idx_file'});
|
||||
($fs->{'last_line'}, $fs->{'last_size'}) = (0, 0);
|
||||
}
|
||||
|
||||
# Skip already processed lines
|
||||
read_snmplogfile($fs) for (1..$fs->{'last_line'});
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Reset the index if the file has been truncated.
|
||||
###############################################################################
|
||||
sub reset_if_truncated($$) {
|
||||
my ($pa_config, $fs) = @_;
|
||||
|
||||
my $log_size = (stat ($fs->{'log_file'}))[7];
|
||||
|
||||
# New SNMP log file found
|
||||
if ($log_size < $fs->{'last_size'}) {
|
||||
logger ($pa_config, 'File ' . $fs->{'log_file'} . ' was truncated.', 10);
|
||||
unlink ($fs->{'idx_file'});
|
||||
($fs->{'last_line'}, $fs->{'last_size'}) = (0, 0);
|
||||
seek($fs->{'fd'}, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Clean-up when the server is destroyed.
|
||||
###############################################################################
|
||||
|
|
|
|||
Loading…
Reference in New Issue