pandorafms/pandora_plugins/Advanced Log Parser/pandora_logparser.pl

772 lines
23 KiB
Perl

#!/usr/bin/perl
###############################################################################
# Pandora FMS Agent Plugin for ADVANCED log parsing
# Copyright (c) 2011-2015 Sancho Lerena <slerena@artica.es>
# Copyright (c) 2011-2015 Artica Soluciones Tecnologicas S.L.
# _______ __ __ _______ _______
# | _ |.----.| |_|__|.----.---.-. | __|_ _|
# | || _|| _| || __| _ | |__ | | |
# |___|___||__| |____|__||____|___._| |_______| |___|
#
#
# ARTICA SOLUCIONES TECNOLOGICAS
# http://www.artica.es
#
# v1r3
# Change log (v1r3 - Ago 2015)
# * Solved lot of issues present in r2
# * Added support for wildcards
# * Identified problem with perl <5.13. It won't work on old Perl. Use binary!
###############################################################################
use strict;
use Data::Dumper;
use File::Basename;
# Used to calculate the MD5 checksum of a string
use constant MOD232 => 2**32;
my @config_file; # Stores the config file contents, line by line
my %plugin_setup; # Hash with this plugin setup
my $archivo_cfg = $ARGV[0]; # External main config file
my $log_items = 0; # Stores total of log definitions in the conf file
my $reg_exp = 0; # Total regexps
my $version;
$version = "v1r3"; # Actual plugin's version
###############################################################################
# SUB load_external_setup
# Receives a configuration filename to load in the configuration hash
###############################################################################
sub load_external_setup ($){
my $archivo_cfg = $_[0];
my $buffer_line;
# Collect items from config file and put in an array
if (! open (CFG, "< $archivo_cfg")) {
print "[ERROR] Error opening configuration file $archivo_cfg: $!.\n";
exit 1;
}
while (<CFG>){
$buffer_line = $_;
# Parse configuration file, this is specially difficult because can contain regexp, with many things
if ($buffer_line !=~ /^\#/){ # begins with anything except # (for commenting)
if ($buffer_line =~ m/(.+)\s(.*)/){
push @config_file, $buffer_line;
}
}
}
close (CFG);
}
###############################################################################
# MD5 leftrotate function. See http://en.wikipedia.org/wiki/MD5#Pseudocode.
###############################################################################
sub leftrotate ($$) {
my ($x, $c) = @_;
return (0xFFFFFFFF & ($x << $c)) | ($x >> (32 - $c));
}
###############################################################################
# Initialize some variables needed by the MD5 algorithm.
# See http://en.wikipedia.org/wiki/MD5#Pseudocode.
###############################################################################
my (@R, @K);
sub md5_init () {
# R specifies the per-round shift amounts
@R = (7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20,
4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21);
# Use binary integer part of the sines of integers (radians) as constants
for (my $i = 0; $i < 64; $i++) {
$K[$i] = floor(abs(sin($i + 1)) * MOD232);
}
}
###############################################################################
# Return the MD5 checksum of the given string.
# Pseudocode from http://en.wikipedia.org/wiki/MD5#Pseudocode.
###############################################################################
sub md5 ($) {
my $str = shift;
# Note: All variables are unsigned 32 bits and wrap modulo 2^32 when calculating
# Initialize variables
my $h0 = 0x67452301;
my $h1 = 0xEFCDAB89;
my $h2 = 0x98BADCFE;
my $h3 = 0x10325476;
# Pre-processing
my $msg = unpack ("B*", pack ("A*", $str));
my $bit_len = length ($msg);
# Append "1" bit to message
$msg .= '1';
# Append "0" bits until message length in bits ≡ 448 (mod 512)
$msg .= '0' while ((length ($msg) % 512) != 448);
# Append bit /* bit, not byte */ length of unpadded message as 64-bit little-endian integer to message
$msg .= unpack ("B64", pack ("VV", $bit_len));
# Process the message in successive 512-bit chunks
for (my $i = 0; $i < length ($msg); $i += 512) {
my @w;
my $chunk = substr ($msg, $i, 512);
# Break chunk into sixteen 32-bit little-endian words w[i], 0 <= i <= 15
for (my $j = 0; $j < length ($chunk); $j += 32) {
push (@w, unpack ("V", pack ("B32", substr ($chunk, $j, 32))));
}
# Initialize hash value for this chunk
my $a = $h0;
my $b = $h1;
my $c = $h2;
my $d = $h3;
my $f;
my $g;
# Main loop
for (my $y = 0; $y < 64; $y++) {
if ($y <= 15) {
$f = $d ^ ($b & ($c ^ $d));
$g = $y;
}
elsif ($y <= 31) {
$f = $c ^ ($d & ($b ^ $c));
$g = (5 * $y + 1) % 16;
}
elsif ($y <= 47) {
$f = $b ^ $c ^ $d;
$g = (3 * $y + 5) % 16;
}
else {
$f = $c ^ ($b | (0xFFFFFFFF & (~ $d)));
$g = (7 * $y) % 16;
}
my $temp = $d;
$d = $c;
$c = $b;
$b = ($b + leftrotate (($a + $f + $K[$y] + $w[$g]) % MOD232, $R[$y])) % MOD232;
$a = $temp;
}
# Add this chunk's hash to result so far
$h0 = ($h0 + $a) % MOD232;
$h1 = ($h1 + $b) % MOD232;
$h2 = ($h2 + $c) % MOD232;
$h3 = ($h3 + $d) % MOD232;
}
# Digest := h0 append h1 append h2 append h3 #(expressed as little-endian)
return unpack ("H*", pack ("V", $h0)) . unpack ("H*", pack ("V", $h1)) . unpack ("H*", pack ("V", $h2)) . unpack ("H*", pack ("V", $h3));
}
###############################################################################
# SUB log_msg
# Print a log message.
###############################################################################
sub log_msg ($) {
my $log_msg = $_[0];
if (! open (LOG, "> ".$plugin_setup{"logfile"})) {
print "[ERROR] Error opening internal logfile ".$plugin_setup{"logfile"}."\n";
exit 1;
}
print LOG $log_msg;
close (LOG);
}
###############################################################################
# SUB error_msg
# Print a log message and exit (fatal log error message)
###############################################################################
sub error_msg ($) {
my $log_msg = $_[0];
log_msg ($log_msg);
}
###############################################################################
# parse_config
#
# This function load configuration tokens and store in a global hash
# called %plugin_setup accesible on all program.
###############################################################################
sub parse_config {
my $tbuf;
my $log_block = 0;
my $reg_exp_rule = 0;
my $parametro;
# Some default options
$plugin_setup{"index_dir"} = "/tmp";
$plugin_setup{"logfile"} = "/tmp/pandora_logparser.log";
$plugin_setup{"log_rotate_mode"}="inode";
foreach (@config_file){
$parametro = $_;
if ($parametro =~ m/^include\s+(.*)$/i) {
load_external_setup ($1);
}
if ($parametro =~ m/^index_dir\s+(.*)$/i) {
$plugin_setup{"index_dir"} = $1;
}
if ($parametro =~ m/^logfile\s+(.*)$/i) {
$plugin_setup{"logfile"} = $1;
}
if ($parametro =~ m/^log\_rotate\_mode\s+(.*)$/i) {
$plugin_setup{"log_rotate_mode"} = $1;
}
# Detect begin of log definition
if ($parametro =~ m/^log\_begin/i) {
$log_block = 1;
}
# Log definition parsing mode
if ($log_block == 1){
if ($parametro =~ m/^log\_module\_name\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"name"} = $1;
}
if ($parametro =~ m/^log\_type\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"type"} = $1;
}
if ($parametro =~ m/^log\_create\_module\_for\_each\_log/i) {
$plugin_setup{"log"}->[$log_items]->{"module_for_each_log"} = 1;
} else {
if (!defined($plugin_setup{"log"}->[$log_items]->{"module_for_each_log"})){
$plugin_setup{"log"}->[$log_items]->{"module_for_each_log"} = 0;
}
}
if ($parametro =~ m/^log\_force\_readall/i) {
$plugin_setup{"log"}->[$log_items]->{"readall"} = 1;
} else {
if (!defined($plugin_setup{"log"}->[$log_items]->{"readall"})){
$plugin_setup{"log"}->[$log_items]->{"readall"} = 0;
}
}
if ($parametro =~ m/^log\_location\_file\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"log_location_file"} = $1;
}
if ($parametro =~ m/^log\_location\_exec\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"log_location_exec"} = $1;
}
if ($parametro =~ m/^log\_location\_multiple\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"log_location_multiple"} = $1;
}
if ($parametro =~ m/^log\_description\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"description"} = $1;
}
if ($parametro =~ m/^log\_regexp\_begin/i) {
$log_block = 2;
$reg_exp_rule = 0;
}
if ($parametro =~ m/^log\_end/i) {
$log_items++;
$log_block = 0;
$reg_exp=0;
}
}
if ($log_block == 2){
if ($parametro =~ m/^log_regexp_severity\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"regexp"}->{$reg_exp}->{"severity"} = $1;
}
if ($parametro =~ m/^log_regexp_rule\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"regexp"}->{$reg_exp}->{"rule"} = $1;
}
if ($parametro =~ m/^log_return_message\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"regexp"}->{$reg_exp} -> {"message"} = $1;
}
if ($parametro =~ m/^log_action\s+(.*)$/i) {
$plugin_setup{"log"}->[$log_items]->{"regexp"}->{$reg_exp} -> {"action"} = $1;
}
if ($parametro =~ m/^log\_regexp\_end/i) {
$log_block = 1;
$reg_exp++;
}
}
}
}
###############################################################################
# clean_blank
#
# This function return a string without blanspaces, given a simple text string
###############################################################################
sub clean_blank($){
my $input = $_[0];
$input =~ s/[\s\r\n]*//g;
return $input;
}
###############################################################################
# SUB load_idx
#
# Load index file and Logfile
###############################################################################
sub load_idx ($$) {
my $Idx_file = $_[0];
my $Log_file = $_[1];
my $line;
my $current_ino;
my $current_size;
my $Idx_pos;
my $Idx_ino;
log_msg("Loading index file $Idx_file");
open(IDXFILE, $Idx_file) || error_msg("Error opening file $Idx_file: " . $!);
# Read position and date
$line = <IDXFILE>;
($Idx_pos, $Idx_ino) = split(' ', $line);
close(IDXFILE);
# Reset the file index if the file has changed
my $current_ino = (stat($Log_file))[1];
my $current_size = (stat($Log_file))[7];
if (($current_ino != $Idx_ino) || ($current_size < $Idx_pos)) {
log_msg("File changed, resetting index");
$Idx_pos = 0;
$Idx_ino = $current_ino;
}
return ($Idx_pos, $Idx_ino);
}
###############################################################################
# SUB save_idx
#
# Save index file, fiven idxfile, logfile, idxpos and idxinode
###############################################################################
sub save_idx ($$$$) {
my $Idx_file = $_[0];
my $Log_file = $_[1];
my $Idx_pos = $_[2];
my $Idx_ino = $_[3];
log_msg("Saving index file $Idx_file");
open(IDXFILE, "> $Idx_file") || error_msg("Error opening file $Idx_file: ". $!);
print (IDXFILE $Idx_pos . " " . $Idx_ino);
close(IDXFILE);
return;
}
###############################################################################
# SUB create_idx
#
# Create index file.
###############################################################################
sub create_idx ($$) {
my $Idx_file = $_[0];
my $Log_file = $_[1];
my $first_line;
log_msg("Creating index file $Idx_file");
open(LOGFILE, $Log_file) || error_msg("Error opening file $Log_file: " . $!);
# Go to EOF and save the position
seek(LOGFILE, 0, 2);
my $Idx_pos = tell(LOGFILE);
close(LOGFILE);
# Save the file inode number
my $Idx_ino = (stat($Log_file))[1];
# Sometimes returns "blank" inode ¿?
if ($Idx_ino eq ""){
$Idx_ino = 0;
}
# Save the index file
save_idx($Idx_file, $Log_file, $Idx_pos, $Idx_ino);
return;
}
###############################################################################
# SUB parse_log
#
# Parse log file starting from position $Idx_pos.
###############################################################################
sub parse_log ($$$$$$$$) {
my $Idx_file = $_[0];
my $Log_file = $_[1];
my $Idx_pos = $_[2];
my $Idx_ino = $_[3];
my $Module_name = $_[4];
my $type = $_[5];
my $regexp_collection = $_[6]; # hash of rules
my $Description = $_[7];
my $line;
my $count = 0;
my $action = "";
my $severity = "";
my $rule = "";
my $buffer = "";
# Parse log file
# Open log file for reading
open(LOGFILE, $Log_file) || error_msg("Error opening file $Log_file: " . $!);
# Go to starting position
seek(LOGFILE, $Idx_pos, 0);
if ($type eq "log_module"){
$buffer = "<log_module>\n";
$buffer .= "<source><![CDATA[" . $Module_name . "]]></source>\n";
$buffer .= "<data><![CDATA[";
} else {
$buffer .= "<module>\n";
$buffer .= "<name><![CDATA[" . $Module_name . "]]></name>\n";
$buffer .= "<description><![CDATA[" . $Description . "]]></description>\n";
if ($type eq "return_ocurrences"){
$buffer .= "<type>generic_data</type>\n";
} else {
$buffer .= "<type><![CDATA[async_string]]></type>\n";
$buffer .= "<datalist>\n";
}
}
while ($line = <LOGFILE>) {
while (my ($key, $value) = each (%{$regexp_collection})) {
# For each regexp block
$rule = $value->{"rule"};
#print "[DEBUG] Action: ".$value->{"action"} ."\n";
#print "[DEBUG] Severity: ".$value->{"severity"} ."\n";
#print "[DEBUG] Message: ".$value->{"message"} ."\n";
#print "[DEBUG] Rule: ".$value->{"rule"} ."\n";
if ($line =~ m/$rule/i) {
# Remove the trailing '\n'
chop($line);
# depending on type:
if ($type eq "return_message"){
$buffer .= "<data><value><![CDATA[".$value->{"message"}."]]></value></data>\n";
}
if ($type eq "return_lines"){
$buffer .= "<data><value><![CDATA[".$line."]]></value></data>\n";
}
if ($type eq "log_module") {
$line =~ s/\]\]/]]]]><![CDATA[/g;
$buffer .= $line."\n";
}
# Critical severity will prevail over other matches
if ($severity eq ""){
$severity = $value->{"severity"};
} elsif ($severity ne "CRITICAL"){
$severity = $value->{"severity"};
}
$action = $value->{"action"};
$count++;
}
}
}
if ($type ne "log_module"){
if ($type eq "return_ocurrences"){
$buffer .= "<data><![CDATA[".$count."]]></data>\n";
} else {
$buffer .= "</datalist>\n";
}
}
# Execute action if any match (always for last match)
if ($count > 0){
`$action`;
}
if ($type ne "log_module"){
# Write severity field in XML
if ($severity ne ""){
$buffer .= "<status>$severity</status>\n";
}
# End XML
$buffer .= "</module>\n";
}else {
$buffer .= "]]></data>\n";
$buffer .= "</log_module>\n";
}
# Update Index
$Idx_pos = tell(LOGFILE);
close(LOGFILE);
# Save the index file
save_idx($Idx_file, $Log_file, $Idx_pos, $Idx_ino);
# There is no need to write XML, no data.
if (($count eq 0) && ($type ne "return_ocurrences")){
return;
}
# Else, show the XML file
print $buffer;
return;
}
###############################################################################
# SUB print_module
#
# Dump a XML module
###############################################################################
sub print_module ($$$$$){
my $MODULE_NAME = $_[0];
my $MODULE_TYPE = $_[1];
my $MODULE_VALUE = $_[2];
my $MODULE_DESC = $_[3];
my $MODULE_STATUS = $_[4];
# If not a string type, remove all blank spaces!
if ($MODULE_TYPE !=~ m/string/){
$MODULE_VALUE = clean_blank($MODULE_VALUE);
}
print "<module>\n";
print "<name>$MODULE_NAME</name>\n";
print "<type>$MODULE_TYPE</type>\n";
print "<data><![CDATA[$MODULE_VALUE]]></data>\n";
if ($MODULE_STATUS ne ""){
print "<status><![CDATA[$MODULE_STATUS]]></status>\n";
}
print "<description><![CDATA[$MODULE_DESC]]></description>\n";
print "</module>\n";
}
###############################################################################
# SUB manage_logfile
#
# Do the stuff with a given file and options
###############################################################################
#manage_logfile($log_filename, $module_name, $readall, $type, $regexp);
sub manage_logfile ($$$$$$){
my $Idx_pos;
my $Idx_ino;
my $Idx_md5;
my $Idx_file;
my $log_filename = $_[0];
my $module_name = $_[1];
my $readall = $_[2];
my $type = $_[3];
my $regexp = $_[4];
my $description = $_[5];
my $index_file_converted = $log_filename;
# Avoid / \ | and : characters
$index_file_converted =~ s/\//_/g;
$index_file_converted =~ s/\\/_/g;
$index_file_converted =~ s/\|/_/g;
$index_file_converted =~ s/\:/_/g;
$module_name =~ s/\//_/g;
$module_name =~ s/\\/_/g;
$module_name =~ s/\|/_/g;
$module_name =~ s/\:/_/g;
# Create index file if it does not exist
if($^O =~ /win/i){
$Idx_file = $plugin_setup{"index_dir"} . "\\". $module_name . "_" . $index_file_converted . "\.idx";
}else{
$Idx_file = $plugin_setup{"index_dir"} . "/". $module_name . "_" . $index_file_converted . "\.idx";
}
# if force read all is enabled,
if (! -e $Idx_file) {
create_idx($Idx_file, $log_filename);
# Load index file
($Idx_pos, $Idx_ino) = load_idx ($Idx_file, $log_filename);
if ($readall == 1){
$Idx_pos = 0;
}
} else {
# Load index file
($Idx_pos, $Idx_ino) = load_idx ($Idx_file, $log_filename);
}
# Parse log file
parse_log($Idx_file, $log_filename, $Idx_pos, $Idx_ino, $module_name, $type, $regexp, $description);
}
###############################################################################
###############################################################################
######################## MAIN PROGRAM CODE ####################################
###############################################################################
###############################################################################
# Parse external configuration file
# -------------------------------------------
# Load config file from command line
if ($#ARGV == -1 ){
print "I need at least one parameter: Complete path to external configuration file \n";
exit;
}
# Check for file
if ( ! -f $archivo_cfg ) {
printf "\n [ERROR] Cannot open configuration file at $archivo_cfg. \n\n";
exit 1;
}
load_external_setup ($archivo_cfg);
parse_config;
#print Dumper(%plugin_setup);
my $log_filename;
my $log_filename_multiple;
my $log_create_module_for_each_log;
my $module_name;
my $module_name_multiple;
my $module_type;
my $readall;
my $type;
my $regexp;
my $description;
# Parse external configuration file
# -------------------------------------------
# Please note that following sentence is not compatible in perl 5.10, it requires
# Perl 5.13 or higher.
while (my ($key, $value) = each (@{$plugin_setup{"log"}})) {
# For each log defined, read data from hash tree
#print "[DEBUG] Key: $key\n";
#print "[DEBUG] Name: ".$value->{name} . "\n";
#print "[DEBUG] Log Location: ".$value->{"log_location_file"} . "\n";
#print "[DEBUG] Log Location exec: ".$value->{"log_location_exec"} . "\n";
#print "[DEBUG] Log Location multiple: ".$value->{"log_location_multiple"} . "\n";
#print "[DEBUG] Log readall: ".$value->{"readall"} . "\n";
#print "[DEBUG] Type: ".$value->{"type"} . "\n";
#print "[DEBUG] Regexp: ".$value->{"regexp"} . "\n";
if (!defined($value->{"name"})) {
print_module ($module_name, "async_string", "", "Missing name in log definition. Skipped", "");
next;
}
$module_name = $value->{"name"};
$readall = $value->{"readall"};
$type = $value->{"type"};
$regexp = $value->{"regexp"};
$description = $value->{"description"};
# Check if filename exists
if (defined($value->{"log_location_file"})){
$log_filename = $value->{"log_location_file"};
manage_logfile ($log_filename, $module_name, $readall, $type, $regexp, $description);
} elsif (defined($value->{"log_location_exec"})){
$log_filename = `$value->{"log_location_exec"}`;
manage_logfile ($log_filename, $module_name, $readall, $type, $regexp, $description);
}
# Multiple files
if (defined($value->{"log_location_multiple"})){
$log_filename_multiple = $value->{"log_location_multiple"};
$log_create_module_for_each_log = $value->{"module_for_each_log"};
my @buffer;
if($^O =~ /win/i){
@buffer = `dir "$log_filename_multiple" /b /a-d`;
}else{
@buffer = `ls -d "$log_filename_multiple"`;
}
foreach (@buffer) {
# This should solve problems with carriage return in Unix, Linux and Windooze
chomp($_);
chop($_) if ($_ =~ m/\r$/);
$log_filename = $_;
$module_name_multiple = $module_name;
if ($log_create_module_for_each_log == 1){
# Create a dynamic module name with the name of the logfile
$module_name_multiple = $log_filename;
$module_name_multiple =~ s/\//_/g;
$module_name_multiple = $module_name . "_" . $module_name_multiple;
}
$log_filename = $log_filename_multiple . $_;
manage_logfile ($log_filename, $module_name_multiple, $readall, $type, $regexp, $description);
}
}
print "\n";
}