tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
pandorafms/pandora_server/util/pandora_db.pl

401 lines
15 KiB
Perl
Executable File
Raw Blame History

#!/usr/bin/perl
##################################################################################
# Pandora FMS DB Management
################################################################################
# Copyright (c) 2004-2006 Sancho Lerena, slerena@gmail.com
# Copyright (c) 2005-2006 Artica Soluciones Tecnologicas S.L
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
################################################################################
# Includes list
use strict;
use Time::Local; # DateTime basic manipulation
use DBI; # DB interface with MySQL
use Date::Manip; # Date/Time manipulation
# version: define la version actual del programa
my $version = "1.3 dev";
# Setup variables
my $dirname="";
my $dbname = 'pandora';
my $dbhost ='';
my $dbuser ='';
my $verbosity ='';
my $dbpass='';
my $server_threshold='';
my $log_file="";
my $pandora_path="";
my $config_days_compact;
my $config_days_purge;
my $config_step_compact;# Step compact variable defines "how-fine" is the compact algorithm. 1 Hour its very fine, 24 hours is bad value
# FLUSH in each IO
$| = 1;
pandora_init();
# Read config file for Global variables
pandora_loadconfig ($pandora_path);
# Begin pandora_server
pandoradb_main();
##################################################################################
##################################################################################
##################################################################################
##################################################################################
##################################################################################
##################################################################################
## SUB pandora_purgedb (days, dbname, dbuser, dbpass, dbhost)
##################################################################################
sub pandora_purgedb {
# 1) Obtain last value for date limit
# 2) Delete all elements below date limit
# 3) Insert last value in date_limit position
my $days = $_[0];
my $dbname = $_[1];
my $dbuser = $_[2];
my $dbpass = $_[3];
my $dbhost = $_[4];
my @query;
my $counter;
my $buffer; my $buffer2; my $buffer3;
my $err; # error code in datecalc function
my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{ RaiseError => 1, AutoCommit => 1 });
# Calculate limit for deletion, today - $days
my $limit_timestamp = DateCalc("today","-$days days",\$err);
my $limit_timestamp2 = DateCalc($limit_timestamp,"+1 minute",\$err);
$limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:%M:%S");
my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S");
my $limit_access = DateCalc("today","-24 hours",\$err);
$limit_access = &UnixDate($limit_access,"%Y-%m-%d %H:%M:%S");
# Lets insert the last value on $limit_timestamp + 1 minute for each id_agente_modulo
my $query_idag = "select count(distinct(id_agente_modulo)) from tagente_datos where timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag);
$idag ->execute;
my @datarow;
@datarow = $idag->fetchrow_array();
if ($verbosity > 0){
print "Total different Modules delete: ".$datarow[0]."\n";
}
my $different_modules = $datarow[0];
$idag->finish;
my $query_idag = "select distinct(id_agente_modulo) from tagente_datos where timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag);
$idag ->execute;
my @datarow;
if ($idag->rows != 0) {
my $counter =0;
while (@datarow = $idag->fetchrow_array()) {
$counter++;
if ($verbosity > 0){
print "\r[PURGEDB] ".$counter / ($different_modules / 100)."% Deleted";
}
$buffer = $datarow[0];
my $query_idag2 = "select * from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer order by timestamp desc limit 1";
my $idag2 = $dbh->prepare($query_idag2);
$idag2 ->execute;
my @datarow2;
if ($idag2->rows != 0) {
while (@datarow2 = $idag2->fetchrow_array()) {
# Create Insert SQL for this data
$buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow2[2],'$limit_timestamp',$datarow2[4])";
}
}
# Execute DELETE
my $query_idag3 = "delete from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer ";
my $idag3 = $dbh->prepare($query_idag3);
$idag3->execute;
$idag3->finish();
# Execute INSERT with last data
my $idag3 = $dbh->prepare($buffer3);
$idag3->execute;
$idag3->finish();
$idag2->finish();
}
}
$idag->finish();
if ($verbosity > 0){
print "[PURGEDB] Deleting static data until $limit_timestamp \n";
}
$query[0] = "delete from tagente_datos_string where timestamp < '$limit_timestamp'";
$query[1] = "delete from tagent_access where timestamp < '$limit_access'";
for ($counter=0;$counter <2; $counter++){
#print "DEBUG SQL Query: $query[$counter] \n";
$dbh->do($query[$a]);
}
$dbh->disconnect();
}
##################################################################################
## SUB pandora_compactdb (days, dbname, dbuser, dbpass, dbhost)
##################################################################################
sub pandora_compactdb {
my $days = $_[0];
my $dbname = $_[1];
my $dbuser = $_[2];
my $dbpass = $_[3];
my $dbhost = $_[4];
my @data_item; # Array to get data from each record of DB
my %data_list; # Hash to store values (sum) for each id
my %data_list_items; # Hash to store total values for each id
my $err; # error code in datecalc function
my $rows_selected = 0; # Calculate how many rows are selected in this loop
my $dbh; # database handler
my $query;
my $query_ready;
my $limit_timestamp; # Define the high limit for timestamp query
my $low_limit; # Define the low limit for timestamp query
my $key; # Used by foreach-loop
my $low_limit_timestamp; # temporal variable to store low limit timestamp
my $oldest_timestamp;
my $flag; #temporal value to store diff between dats
# Begin procedure (SQL open initizalizacion and initial timestamp calculation)
$dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{ RaiseError => 1, AutoCommit => 1 });
# Get the first (oldest) timestamp in database to make it the marker for the end of query
$query = "select min(timestamp) from tagente_datos ";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
@data_item = $query_ready->fetchrow_array();
$oldest_timestamp = @data_item[0];
$query_ready->finish;
# We need to determine data ranges
# Calculate start limit for compactation, Today- X hour to older datetime
$limit_timestamp = DateCalc("today","-$days days",\$err);
$limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:00:00");
print "[COMPACT] Packing data from $limit_timestamp to $oldest_timestamp \n";
# Main loop
do { # Makes a query for each days from $limit_timestamp (now is set $days from today)
# To get actual low limit, minus step_compact hours
$low_limit_timestamp = DateCalc("$limit_timestamp","-$config_step_compact hours",\$err);
$low_limit_timestamp = &UnixDate($low_limit_timestamp,"%Y-%m-%d %H:%M:%S");
if ($verbosity > 0){
print "[COMPACT] Working at interval: $limit_timestamp - $low_limit_timestamp \n";
}
# DB Query to get data from DB based on timestamp limits
$query = "select * from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
if ($rows_selected > 0) {
# Init hashes
%data_list=();
%data_list_items=();
# Create a hash data_list with id_agente_mopulo as index
# and creating a summatory of values for this inverval
# storing the total number of dif. values in data_list_items hash
while (@data_item = $query_ready->fetchrow_array()) {
$data_list{$data_item[1]}= $data_list{$data_item[1]} + $data_item[2];
$data_list_items{$data_item[1]}=$data_list_items{$data_item[1]} + 1;
}
# Once we has filled the hast, let's delete records processed for this
# interval. Later we could insert the new record, and initialize hash for
# reuse it in the next loop.
$query = "delete from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp' ";
$dbh->do($query);
# print "DEBUG: Purge query $query \n";
my $value; my $value_timestamp;
foreach $key (keys (%data_list)) {
$value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-)
$query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')";
$dbh->do($query);
if ($verbosity > 0){
print "DEBUG: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n";
}
# Purge hash
delete $data_list{$key};
delete $data_list_items{$key};
}
}
$limit_timestamp = $low_limit_timestamp; # To the next day !!
$flag = Date_Cmp($oldest_timestamp,$limit_timestamp);
} until ($flag >= 0);
$query_ready->finish();
$dbh->disconnect();
}
##############################################################################
# SUB pandora_init ()
# Makes the initial parameter parsing, initializing and error checking
##############################################################################
sub pandora_init {
printf "\nPandora DB $version, <slerena\@gmail.com>\n";
printf "This program is GPL Licensed, please use it freely\n";
# Load config file from command line
if ($#ARGV == -1 ){
print "FATAL ERROR: I Need at least one parameter: Complete path to pandora_server.conf file !!\n\n";
exit;
}
# If there are not valid parameters
my $parametro;
my $ltotal=$#ARGV; my $ax;
for ($ax=0;$ax<=$ltotal;$ax++){
$parametro = $ARGV[$ax];
if ($parametro =~ m/-h\z/i ) { help_screen(); }
elsif ($parametro =~ m/-help\z/i ) { help_screen(); }
elsif ($parametro =~ m/--help\z/i ) { help_screen(); }
elsif ($parametro =~ m/-v\z/i) { $verbosity=5; }
elsif ($parametro =~ m/-d\z/i) { $verbosity=10; }
else { ($pandora_path = $parametro); }
}
if ($pandora_path eq ""){
print "FATAL ERROR: I Need complete path to pandora_server.conf file !!\n\n";
exit;
}
}
##############################################################################
# Read external configuration file
##############################################################################
sub pandora_loadconfig {
my $archivo_cfg = @_[0];
my $buffer_line;
my @command_line;
# Check for file
if ( ! -e $archivo_cfg ) {
printf "[ERROR] Cannot open configuration file. Please specify a valid one in command line \n";
exit 1;
}
# Collect items from config file and put in an array
open (CFG, "< $archivo_cfg");
while (<CFG>){
$buffer_line = $_;
if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\!\*\@\#\%\$\~\"\']+)/){
push @command_line,$1;
push @command_line,$2;
}
}
close (CFG);
# Process this array with commandline like options
# Process input parameters
my @args = @command_line;
my $parametro;
my $ltotal=$#args; my $ax;
# Has read setup file ok ?
if ( $ltotal == 0 ) {
print "[ERROR] No valid setup tokens readed in $archivo_cfg ";
exit;
}
for ($ax=0;$ax<=$ltotal;$ax++){
$parametro = $args[$ax];
if ($parametro =~ m/dirname\z/) { $dirname = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/dbuser\z/) { $dbuser = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/dbpass\z/) { $dbpass = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/dbname\z/) { $dbname = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/dbhost\z/) { $dbhost = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/log_file\z/) { $log_file = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/verbosity\z/) { $verbosity = $args[$ax+1]; $ax++; }
elsif ($parametro =~ m/server_threshold\z/) { $server_threshold = $args[$ax+1]; $ax++; }
}
# Check for valid token token values
if (( $dbuser eq "" ) || ( $log_file eq "" ) || ( $dbhost eq "") || ( $dbpass eq "" ) ) {
print "[ERROR] Bad Config values. Be sure that $archivo_cfg is a valid setup file";
exit;
}
# Open database to get days_purge days_compact values
my $query; my $query_ready; my @data; my $rows_selected;
my $dbh = DBI->connect("DBI:mysql:pandora:$dbhost:3306",$dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 });
$query = "select * from tconfig where token = 'days_purge'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
if ($query_ready->rows > 0) {
@data = $query_ready->fetchrow_array();
$config_days_purge = $data[2]; # value
} else {
print "[ERROR] I cannot find in database a config item (DAYS_PURGE)\n";
exit(-1);
}
$query_ready->finish();
$query = "select * from tconfig where token = 'days_compact'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
if ($query_ready->rows > 0) {
@data = $query_ready->fetchrow_array();
$config_days_compact = $data[2]; # value
} else {
print "[ERROR] I cannot find in database a config item (DAYS_COMPACT)\n";
exit(-1);
}
$query_ready->finish();
$query = "select * from tconfig where token = 'step_compact'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
if ($query_ready->rows > 0) {
@data = $query_ready->fetchrow_array();
$config_step_compact = $data[2]; # value
} else {
print "[ERROR] I cannot find in database a config item (CONFIG_STEP_COMPACT)\n";
exit(-1);
}
$query_ready->finish();
$dbh->disconnect;
printf "Pandora DB now initialized and running (PURGE=$config_days_purge days, COMPACT=$config_days_compact days, STEP=$config_step_compact) ... \n\n";
}
##############################################################################
# SUB help_screen()
# Show a help screen an exits
##############################################################################
sub help_screen{
printf "\n\nSintax: \n pandora_db.pl fullpathname_to_pandora_server.conf \n\n";
exit;
}
# #################################################################################
# Program main begin
# #################################################################################
sub pandoradb_main {
pandora_purgedb ($config_days_purge, $dbname, $dbuser, $dbpass, $dbhost);
pandora_compactdb ($config_days_compact, $dbname, $dbuser, $dbpass, $dbhost);
exit;
}
Reference in New Issue View Git Blame Copy Permalink