tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

2007-08-28 Sancho Lerena <slerena@gmail.com> 

* util/pandora_db.pl: Added some fixes and new checks for database
        consistency. Also purge non-init modules (needed to avoid unused modules
        that take too much time on network servers!).



git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@627 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f
This commit is contained in:
slerena 2007-08-28 15:09:04 +00:00
parent 1f9f1ef933
commit 0d0d5a1c59
2 changed files with 195 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pandora_server/ChangeLog
View File

@ -1,3 +1,9 @@
2007-08-28 Sancho Lerena <slerena@gmail.com>
* util/pandora_db.pl: Added some fixes and new checks for database
consistency. Also purge non-init modules (needed to avoid unused modules
that take too much time on network servers!).
2007-08-27 Sancho Lerena <slerena@gmail.com>
* lib/PandoraFMS/Config.pm: Updated version string.

265
pandora_server/util/pandora_db.pl
View File

@ -1,8 +1,8 @@
#!/usr/bin/perl
##################################################################################
###############################################################################
# Pandora FMS DB Management
################################################################################
###############################################################################
# Copyright (c) 2004-2006 Sancho Lerena, slerena@gmail.com
# Copyright (c) 2005-2006 Artica Soluciones Tecnologicas S.L
#
@ -15,8 +15,8 @@
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
################################################################################
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA
###############################################################################
# Includes list
use strict;
@ -25,7 +25,7 @@ use DBI; # DB interface with MySQL
use Date::Manip; # Date/Time manipulation
# version: define la version actual del programa
my $version = "1.3 dev";
my $version = "1.3 PS070828";
# Setup variables
my $dirname="";
@ -39,7 +39,7 @@ my $log_file="";
my $pandora_path="";
my $config_days_compact;
my $config_days_purge;
my $config_step_compact;# Step compact variable defines "how-fine" is the compact algorithm. 1 Hour its very fine, 24 hours is bad value
my $config_step_compact;# Step compact variable defines "how-fine" is thecompact algorithm. 1 Hour its very fine, 24 hours is bad value
# FLUSH in each IO
$| = 1;
@ -52,15 +52,15 @@ pandora_loadconfig ($pandora_path);
# Begin pandora_server
pandoradb_main();
##################################################################################
##################################################################################
##################################################################################
##################################################################################
##################################################################################
###############################################################################
###############################################################################
###############################################################################
###############################################################################
###############################################################################
##################################################################################
###############################################################################
## SUB pandora_purgedb (days, dbname, dbuser, dbpass, dbhost)
##################################################################################
###############################################################################
sub pandora_purgedb {
# 1) Obtain last value for date limit
@ -76,14 +76,15 @@ sub pandora_purgedb {
my $counter;
my $buffer; my $buffer2; my $buffer3;
my $err; # error code in datecalc function
my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{ RaiseError => 1, AutoCommit => 1 });
my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{RaiseError => 1, AutoCommit => 1 });
# Calculate limit for deletion, today - $days
my $limit_timestamp = DateCalc("today","-$days days",\$err);
my $limit_timestamp2 = DateCalc($limit_timestamp,"+1 minute",\$err);
$limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:%M:%S");
my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S");
my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S");
my $limit_access = DateCalc("today","-24 hours",\$err);
$limit_access = &UnixDate($limit_access,"%Y-%m-%d %H:%M:%S");
print "[PURGE] Deleting old data... \n";
# Lets insert the last value on $limit_timestamp + 1 minute for each id_agente_modulo
my $query_idag = "select count(distinct(id_agente_modulo)) from tagente_datos where timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag);
@ -91,12 +92,12 @@ sub pandora_purgedb {
my @datarow;
@datarow = $idag->fetchrow_array();
if ($verbosity > 0){
print "Total different Modules delete: ".$datarow[0]."\n";
print "[PURGE] Total different Modules delete: ".$datarow[0]."\n";
}
my $different_modules = $datarow[0];
$idag->finish;
my $query_idag = "select distinct(id_agente_modulo) from tagente_datos where timestamp < '$limit_timestamp'";
my $query_idag = "select distinct(id_agente_modulo) from tagente_datos WHERE timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag);
$idag ->execute;
my @datarow;
@ -105,7 +106,7 @@ sub pandora_purgedb {
while (@datarow = $idag->fetchrow_array()) {
$counter++;
if ($verbosity > 0){
print "\r[PURGEDB] ".$counter / ($different_modules / 100)."% Deleted";
print "\r[PURGE] ".$counter / ($different_modules / 100)."% Deleted";
}
$buffer = $datarow[0];
my $query_idag2 = "select * from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer order by timestamp desc limit 1";
@ -116,7 +117,7 @@ sub pandora_purgedb {
if ($idag2->rows != 0) {
while (@datarow2 = $idag2->fetchrow_array()) {
# Create Insert SQL for this data
$buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow2[2],'$limit_timestamp',$datarow2[4])";
$buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow[2],'$limit_timestamp',$datarow2[4])";
}
}
# Execute DELETE
@ -135,7 +136,7 @@ sub pandora_purgedb {
}
$idag->finish();
if ($verbosity > 0){
print "[PURGEDB] Deleting static data until $limit_timestamp \n";
print "[PURGE] Deleting static data until $limit_timestamp \n";
}
$query[0] = "delete from tagente_datos_string where timestamp < '$limit_timestamp'";
$query[1] = "delete from tagent_access where timestamp < '$limit_access'";
@ -146,9 +147,9 @@ sub pandora_purgedb {
$dbh->disconnect();
}
##################################################################################
###############################################################################
## SUB pandora_compactdb (days, dbname, dbuser, dbpass, dbhost)
##################################################################################
###############################################################################
sub pandora_compactdb {
my $days = $_[0];
my $dbname = $_[1];
@ -176,69 +177,74 @@ sub pandora_compactdb {
# Get the first (oldest) timestamp in database to make it the marker for the end of query
$query = "select min(timestamp) from tagente_datos ";
$query = "SELECT min(timestamp) FROM tagente_datos ";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
@data_item = $query_ready->fetchrow_array();
$oldest_timestamp = @data_item[0];
$query_ready->finish;
# We need to determine data ranges
# Calculate start limit for compactation, Today- X hour to older datetime
$limit_timestamp = DateCalc("today","-$days days",\$err);
$limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:00:00");
print "[COMPACT] Packing data from $limit_timestamp to $oldest_timestamp \n";
# If no data, skip this step
if ($oldest_timestamp != ""){
# We need to determine data ranges
# Calculate start limit for compactation, Today- X hour to older datetime
$limit_timestamp = DateCalc("today","-$days days",\$err);
$limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:00:00");
print "[COMPACT] Packing data from $limit_timestamp to $oldest_timestamp \n";
# Main loop
do { # Makes a query for each days from $limit_timestamp (now is set $days from today)
# To get actual low limit, minus step_compact hours
$low_limit_timestamp = DateCalc("$limit_timestamp","-$config_step_compact hours",\$err);
$low_limit_timestamp = &UnixDate($low_limit_timestamp,"%Y-%m-%d %H:%M:%S");
if ($verbosity > 0){
print "[COMPACT] Working at interval: $limit_timestamp - $low_limit_timestamp \n";
}
# DB Query to get data from DB based on timestamp limits
$query = "select * from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
if ($rows_selected > 0) {
# Init hashes
%data_list=();
%data_list_items=();
# Create a hash data_list with id_agente_mopulo as index
# and creating a summatory of values for this inverval
# storing the total number of dif. values in data_list_items hash
while (@data_item = $query_ready->fetchrow_array()) {
$data_list{$data_item[1]}= $data_list{$data_item[1]} + $data_item[2];
$data_list_items{$data_item[1]}=$data_list_items{$data_item[1]} + 1;
# Main loop
do { # Makes a query for each days from $limit_timestamp (now is set $days from today)
# To get actual low limit, minus step_compact hours
$low_limit_timestamp = DateCalc("$limit_timestamp","-$config_step_compact hours",\$err);
$low_limit_timestamp = &UnixDate($low_limit_timestamp,"%Y-%m-%d %H:%M:%S");
if ($verbosity > 0){
print "[COMPACT] Working at interval: $limit_timestamp -$low_limit_timestamp \n";
}
# Once we has filled the hast, let's delete records processed for this
# interval. Later we could insert the new record, and initialize hash for
# reuse it in the next loop.
$query = "delete from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp' ";
$dbh->do($query);
# DB Query to get data from DB based on timestamp limits
$query = "select * from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
$rows_selected = $query_ready->rows;
# print "DEBUG: Purge query $query \n";
my $value; my $value_timestamp;
foreach $key (keys (%data_list)) {
$value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-)
$query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')";
$dbh->do($query);
if ($verbosity > 0){
print "DEBUG: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n";
if ($rows_selected > 0) {
# Init hashes
%data_list=();
%data_list_items=();
# Create a hash data_list with id_agente_mopulo as index
# and creating a summatory of values for this inverval
# storing the total number of dif. values in data_list_items hash
while (@data_item = $query_ready->fetchrow_array()) {
$data_list{$data_item[1]}= $data_list{$data_item[1]} + $data_item[2];
$data_list_items{$data_item[1]}=$data_list_items{$data_item[1]} + 1;
}
# Once we has filled the hast, let's delete records processed for this
# interval. Later we could insert the new record, and initialize hash for
# reuse it in the next loop.
$query = "delete from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp' ";
$dbh->do($query);
# print "DEBUG: Purge query $query \n";
my $value; my $value_timestamp;
foreach $key (keys (%data_list)) {
$value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-)
$query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')";
$dbh->do($query);
#if ($verbosity > 0){
# print "[DEBUG]: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n";
#}
# Purge hash
delete $data_list{$key};
delete $data_list_items{$key};
}
# Purge hash
delete $data_list{$key};
delete $data_list_items{$key};
}
}
$limit_timestamp = $low_limit_timestamp; # To the next day !!
$flag = Date_Cmp($oldest_timestamp,$limit_timestamp);
} until ($flag >= 0);
$limit_timestamp = $low_limit_timestamp; # To the next day !!
$flag = Date_Cmp($oldest_timestamp,$limit_timestamp);
} until ($flag >= 0);
} else {
print "[COMPACT] No data to pack ! \n";
}
$query_ready->finish();
$dbh->disconnect();
}
@ -249,9 +255,9 @@ sub pandora_compactdb {
##############################################################################
sub pandora_init {
print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 ArticaST\n";
print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 Sancho Lerena\n";
print "This program is Free Software, licensed under the terms of GPL License v2\n";
print "You can download latest versions and documentation at http://pandora.sourceforge.net\n";
print "You can download latest versions and documentation at http://pandora.sf.net\n";
# Load config file from command line
if ($#ARGV == -1 ){
@ -297,7 +303,7 @@ sub pandora_loadconfig {
open (CFG, "< $archivo_cfg");
while (<CFG>){
$buffer_line = $_;
if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\!\*\@\#\%\$\~\"\']+)/){
if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\\*\@\#\%\$\~\"\']+)/){
push @command_line,$1;
push @command_line,$2;
}
@ -329,14 +335,14 @@ sub pandora_loadconfig {
}
# Check for valid token token values
if (( $dbuser eq "" ) || ( $log_file eq "" ) || ( $dbhost eq "") || ( $dbpass eq "" ) ) {
if (( $dbuser eq "" ) || ( $log_file eq "" ) || ( $dbhost eq "") || ($dbpass eq "" ) ) {
print "[ERROR] Bad Config values. Be sure that $archivo_cfg is a valid setup file";
exit;
}
# Open database to get days_purge days_compact values
my $query; my $query_ready; my @data; my $rows_selected;
my $dbh = DBI->connect("DBI:mysql:pandora:$dbhost:3306",$dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 });
my $dbh = DBI->connect("DBI:mysql:pandora:$dbhost:3306",$dbuser, $dbpass, {RaiseError => 1, AutoCommit => 1 });
$query = "select * from tconfig where token = 'days_purge'";
$query_ready = $dbh->prepare($query);
$query_ready ->execute();
@ -381,6 +387,101 @@ sub pandora_loadconfig {
printf "Pandora DB now initialized and running (PURGE=$config_days_purge days, COMPACT=$config_days_compact days, STEP=$config_step_compact) ... \n\n";
}
###############################################################################
## SUB pandora_checkdb_consistency (dbname, dbuser, dbpass, dbhost)
###############################################################################
sub pandora_checkdb_consistency {
# 1. Check for modules that do not have tagente_estado but have tagente_module
my $dbname = $_[0];
my $dbuser = $_[1];
my $dbpass = $_[2];
my $dbhost = $_[3];
my @query;
my $counter;
my $err; # error code in datecalc function
my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{RaiseError => 1, AutoCommit => 1 });
print "[CHECKDB] Checking database consistency (step1)... \n";
my $query1 = "SELECT * FROM tagente_modulo";
my $prep1 = $dbh->prepare($query1);
$prep1 ->execute;
my @datarow1;
if ($prep1->rows != 0) {
# for each record in tagente_modulo
while (@datarow1 = $prep1->fetchrow_array()) {
my $id_agente_modulo = $datarow1[0];
# check if exist in tagente_estado and create if not
my $query2 = "SELECT * FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo";
my $prep2 = $dbh->prepare($query2);
$prep2->execute;
# If have 0 items, we need to create tagente_estado record
if ($prep2->rows == 0) {
my $id_agente = $datarow1[1];
my $query3 = "INSERT INTO tagente_estado (id_agente_modulo, datos, timestamp, cambio, estado, id_agente, last_try, utimestamp, current_interval, running_by, last_execution_try) VALUE ($id_agente_modulo, 0, '0000-00-00 00:00:00', 0, 100, $id_agente, '0000-00-00 00:00:00', 0, 0, 0, 0)";
print "[CHECKDB] Inserting module $id_agente_modulo in state table \n";
my $prep3 = $dbh->prepare($query3);
$prep3->execute;
$prep3->finish();
}
$prep2->finish();
}
}
$prep1->finish();
print "[CHECKDB] Checking database consistency (step2)... \n";
# 2. Check for modules in tagente_estado that do not have tagente_modulo, if there is any, delete it
my $query1 = "SELECT * FROM tagente_estado";
my $prep1 = $dbh->prepare($query1);
$prep1 ->execute;
my @datarow1;
if ($prep1->rows != 0) {
# for each record in tagente_modulo
while (@datarow1 = $prep1->fetchrow_array()) {
my $id_agente_modulo = $datarow1[1];
# check if exist in tagente_estado and create if not
my $query2 = "SELECT * FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo";
my $prep2 = $dbh->prepare($query2);
$prep2->execute;
# If have 0 items, we need to create tagente_estado record
if ($prep2->rows == 0) {
my $id_agente = $datarow1[1];
my $query3 = "DELETE FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo";
print "[CHECKDB] Deleting non-existing module $id_agente_modulo in state table \n";
my $prep3 = $dbh->prepare($query3);
$prep3->execute;
$prep3->finish();
}
$prep2->finish();
}
}
$prep1->finish();
print "[CHECKDB] Deleting non-init data... \n";
my $query4 = "SELECT * FROM tagente_estado WHERE utimestamp = 0";
my $prep4 = $dbh->prepare($query4);
$prep4 ->execute;
my @datarow4;
if ($prep4->rows != 0) {
# for each record in tagente_modulo
while (@datarow4 = $prep4->fetchrow_array()) {
my $id_agente_modulo = $datarow4[1];
my $query0 = "DELETE FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo";
my $prep0 = $dbh->prepare($query0);
$prep0 ->execute;
$prep0->finish();
}
}
$prep4->finish();
# Delete from tagente_estado
my $query0 = "DELETE FROM tagente_estado WHERE utimestamp = 0";
my $prep0 = $dbh->prepare($query0);
$prep0 ->execute;
$prep0->finish();
}
##############################################################################
# SUB help_screen()
# Show a help screen an exits
@ -394,11 +495,15 @@ sub help_screen{
exit;
}
# #################################################################################
#
###############################################################################
# Program main begin
# #################################################################################
#
###############################################################################
sub pandoradb_main {
pandora_purgedb ($config_days_purge, $dbname, $dbuser, $dbpass, $dbhost);
pandora_compactdb ($config_days_compact, $dbname, $dbuser, $dbpass, $dbhost);
pandora_checkdb_consistency ($dbname, $dbuser, $dbpass, $dbhost);
print "\n";
exit;
}