diff --git a/pandora_server/ChangeLog b/pandora_server/ChangeLog index 94b6d3506f..22c5a5f750 100644 --- a/pandora_server/ChangeLog +++ b/pandora_server/ChangeLog @@ -1,3 +1,9 @@ +2007-08-28 Sancho Lerena + + * util/pandora_db.pl: Added some fixes and new checks for database + consistency. Also purge non-init modules (needed to avoid unused modules + that take too much time on network servers!). + 2007-08-27 Sancho Lerena * lib/PandoraFMS/Config.pm: Updated version string. diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index 480b8edcf9..77ffab7dbe 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -1,8 +1,8 @@ #!/usr/bin/perl -################################################################################## +############################################################################### # Pandora FMS DB Management -################################################################################ +############################################################################### # Copyright (c) 2004-2006 Sancho Lerena, slerena@gmail.com # Copyright (c) 2005-2006 Artica Soluciones Tecnologicas S.L # @@ -15,8 +15,8 @@ # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -################################################################################ +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA +############################################################################### # Includes list use strict; @@ -25,7 +25,7 @@ use DBI; # DB interface with MySQL use Date::Manip; # Date/Time manipulation # version: define la version actual del programa -my $version = "1.3 dev"; +my $version = "1.3 PS070828"; # Setup variables my $dirname=""; @@ -39,7 +39,7 @@ my $log_file=""; my $pandora_path=""; my $config_days_compact; my $config_days_purge; -my $config_step_compact;# Step compact variable defines "how-fine" is the compact algorithm. 1 Hour its very fine, 24 hours is bad value +my $config_step_compact;# Step compact variable defines "how-fine" is thecompact algorithm. 1 Hour its very fine, 24 hours is bad value # FLUSH in each IO $| = 1; @@ -52,15 +52,15 @@ pandora_loadconfig ($pandora_path); # Begin pandora_server pandoradb_main(); -################################################################################## -################################################################################## -################################################################################## -################################################################################## -################################################################################## +############################################################################### +############################################################################### +############################################################################### +############################################################################### +############################################################################### -################################################################################## +############################################################################### ## SUB pandora_purgedb (days, dbname, dbuser, dbpass, dbhost) -################################################################################## +############################################################################### sub pandora_purgedb { # 1) Obtain last value for date limit @@ -76,14 +76,15 @@ sub pandora_purgedb { my $counter; my $buffer; my $buffer2; my $buffer3; my $err; # error code in datecalc function - my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{ RaiseError => 1, AutoCommit => 1 }); + my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{RaiseError => 1, AutoCommit => 1 }); # Calculate limit for deletion, today - $days my $limit_timestamp = DateCalc("today","-$days days",\$err); my $limit_timestamp2 = DateCalc($limit_timestamp,"+1 minute",\$err); $limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:%M:%S"); - my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S"); + my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S"); my $limit_access = DateCalc("today","-24 hours",\$err); $limit_access = &UnixDate($limit_access,"%Y-%m-%d %H:%M:%S"); + print "[PURGE] Deleting old data... \n"; # Lets insert the last value on $limit_timestamp + 1 minute for each id_agente_modulo my $query_idag = "select count(distinct(id_agente_modulo)) from tagente_datos where timestamp < '$limit_timestamp'"; my $idag = $dbh->prepare($query_idag); @@ -91,12 +92,12 @@ sub pandora_purgedb { my @datarow; @datarow = $idag->fetchrow_array(); if ($verbosity > 0){ - print "Total different Modules delete: ".$datarow[0]."\n"; + print "[PURGE] Total different Modules delete: ".$datarow[0]."\n"; } my $different_modules = $datarow[0]; $idag->finish; - my $query_idag = "select distinct(id_agente_modulo) from tagente_datos where timestamp < '$limit_timestamp'"; + my $query_idag = "select distinct(id_agente_modulo) from tagente_datos WHERE timestamp < '$limit_timestamp'"; my $idag = $dbh->prepare($query_idag); $idag ->execute; my @datarow; @@ -105,7 +106,7 @@ sub pandora_purgedb { while (@datarow = $idag->fetchrow_array()) { $counter++; if ($verbosity > 0){ - print "\r[PURGEDB] ".$counter / ($different_modules / 100)."% Deleted"; + print "\r[PURGE] ".$counter / ($different_modules / 100)."% Deleted"; } $buffer = $datarow[0]; my $query_idag2 = "select * from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer order by timestamp desc limit 1"; @@ -116,7 +117,7 @@ sub pandora_purgedb { if ($idag2->rows != 0) { while (@datarow2 = $idag2->fetchrow_array()) { # Create Insert SQL for this data - $buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow2[2],'$limit_timestamp',$datarow2[4])"; + $buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow[2],'$limit_timestamp',$datarow2[4])"; } } # Execute DELETE @@ -135,7 +136,7 @@ sub pandora_purgedb { } $idag->finish(); if ($verbosity > 0){ - print "[PURGEDB] Deleting static data until $limit_timestamp \n"; + print "[PURGE] Deleting static data until $limit_timestamp \n"; } $query[0] = "delete from tagente_datos_string where timestamp < '$limit_timestamp'"; $query[1] = "delete from tagent_access where timestamp < '$limit_access'"; @@ -146,9 +147,9 @@ sub pandora_purgedb { $dbh->disconnect(); } -################################################################################## +############################################################################### ## SUB pandora_compactdb (days, dbname, dbuser, dbpass, dbhost) -################################################################################## +############################################################################### sub pandora_compactdb { my $days = $_[0]; my $dbname = $_[1]; @@ -176,69 +177,74 @@ sub pandora_compactdb { # Get the first (oldest) timestamp in database to make it the marker for the end of query - $query = "select min(timestamp) from tagente_datos "; + $query = "SELECT min(timestamp) FROM tagente_datos "; $query_ready = $dbh->prepare($query); $query_ready ->execute(); @data_item = $query_ready->fetchrow_array(); $oldest_timestamp = @data_item[0]; $query_ready->finish; - - # We need to determine data ranges - # Calculate start limit for compactation, Today- X hour to older datetime - $limit_timestamp = DateCalc("today","-$days days",\$err); - $limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:00:00"); - print "[COMPACT] Packing data from $limit_timestamp to $oldest_timestamp \n"; - - # Main loop - do { # Makes a query for each days from $limit_timestamp (now is set $days from today) - # To get actual low limit, minus step_compact hours - $low_limit_timestamp = DateCalc("$limit_timestamp","-$config_step_compact hours",\$err); - $low_limit_timestamp = &UnixDate($low_limit_timestamp,"%Y-%m-%d %H:%M:%S"); - if ($verbosity > 0){ - print "[COMPACT] Working at interval: $limit_timestamp - $low_limit_timestamp \n"; - } - # DB Query to get data from DB based on timestamp limits - $query = "select * from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp'"; - $query_ready = $dbh->prepare($query); - $query_ready ->execute(); - $rows_selected = $query_ready->rows; - - if ($rows_selected > 0) { - # Init hashes - %data_list=(); - %data_list_items=(); - # Create a hash data_list with id_agente_mopulo as index - # and creating a summatory of values for this inverval - # storing the total number of dif. values in data_list_items hash - while (@data_item = $query_ready->fetchrow_array()) { - $data_list{$data_item[1]}= $data_list{$data_item[1]} + $data_item[2]; - $data_list_items{$data_item[1]}=$data_list_items{$data_item[1]} + 1; + # If no data, skip this step + if ($oldest_timestamp != ""){ + # We need to determine data ranges + # Calculate start limit for compactation, Today- X hour to older datetime + $limit_timestamp = DateCalc("today","-$days days",\$err); + $limit_timestamp = &UnixDate($limit_timestamp,"%Y-%m-%d %H:00:00"); + print "[COMPACT] Packing data from $limit_timestamp to $oldest_timestamp \n"; + + # Main loop + do { # Makes a query for each days from $limit_timestamp (now is set $days from today) + # To get actual low limit, minus step_compact hours + $low_limit_timestamp = DateCalc("$limit_timestamp","-$config_step_compact hours",\$err); + $low_limit_timestamp = &UnixDate($low_limit_timestamp,"%Y-%m-%d %H:%M:%S"); + if ($verbosity > 0){ + print "[COMPACT] Working at interval: $limit_timestamp -$low_limit_timestamp \n"; } - # Once we has filled the hast, let's delete records processed for this - # interval. Later we could insert the new record, and initialize hash for - # reuse it in the next loop. - - $query = "delete from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp' "; - $dbh->do($query); - - # print "DEBUG: Purge query $query \n"; - my $value; my $value_timestamp; - foreach $key (keys (%data_list)) { - $value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-) - $query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')"; - $dbh->do($query); - if ($verbosity > 0){ - print "DEBUG: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n"; + + # DB Query to get data from DB based on timestamp limits + $query = "select * from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp'"; + $query_ready = $dbh->prepare($query); + $query_ready ->execute(); + $rows_selected = $query_ready->rows; + + if ($rows_selected > 0) { + # Init hashes + %data_list=(); + %data_list_items=(); + # Create a hash data_list with id_agente_mopulo as index + # and creating a summatory of values for this inverval + # storing the total number of dif. values in data_list_items hash + while (@data_item = $query_ready->fetchrow_array()) { + $data_list{$data_item[1]}= $data_list{$data_item[1]} + $data_item[2]; + $data_list_items{$data_item[1]}=$data_list_items{$data_item[1]} + 1; + } + # Once we has filled the hast, let's delete records processed for this + # interval. Later we could insert the new record, and initialize hash for + # reuse it in the next loop. + + $query = "delete from tagente_datos where timestamp < '$limit_timestamp' and timestamp >= '$low_limit_timestamp' "; + $dbh->do($query); + + # print "DEBUG: Purge query $query \n"; + my $value; my $value_timestamp; + foreach $key (keys (%data_list)) { + $value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-) + $query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')"; + $dbh->do($query); + #if ($verbosity > 0){ + # print "[DEBUG]: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n"; + #} + # Purge hash + delete $data_list{$key}; + delete $data_list_items{$key}; } - # Purge hash - delete $data_list{$key}; - delete $data_list_items{$key}; } - } - $limit_timestamp = $low_limit_timestamp; # To the next day !! - $flag = Date_Cmp($oldest_timestamp,$limit_timestamp); - } until ($flag >= 0); + $limit_timestamp = $low_limit_timestamp; # To the next day !! + $flag = Date_Cmp($oldest_timestamp,$limit_timestamp); + } until ($flag >= 0); + } else { + print "[COMPACT] No data to pack ! \n"; + } $query_ready->finish(); $dbh->disconnect(); } @@ -249,9 +255,9 @@ sub pandora_compactdb { ############################################################################## sub pandora_init { - print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 ArticaST\n"; + print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 Sancho Lerena\n"; print "This program is Free Software, licensed under the terms of GPL License v2\n"; - print "You can download latest versions and documentation at http://pandora.sourceforge.net\n"; + print "You can download latest versions and documentation at http://pandora.sf.net\n"; # Load config file from command line if ($#ARGV == -1 ){ @@ -297,7 +303,7 @@ sub pandora_loadconfig { open (CFG, "< $archivo_cfg"); while (){ $buffer_line = $_; - if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\!\*\@\#\%\$\~\"\']+)/){ + if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\\*\@\#\%\$\~\"\']+)/){ push @command_line,$1; push @command_line,$2; } @@ -325,18 +331,18 @@ sub pandora_loadconfig { elsif ($parametro =~ m/dbname\z/) { $dbname = $args[$ax+1]; $ax++; } elsif ($parametro =~ m/dbhost\z/) { $dbhost = $args[$ax+1]; $ax++; } elsif ($parametro =~ m/log_file\z/) { $log_file = $args[$ax+1]; $ax++; } - elsif ($parametro =~ m/server_threshold\z/) { $server_threshold = $args[$ax+1]; $ax++; } + elsif ($parametro =~ m/server_threshold\z/) { $server_threshold = $args[$ax+1]; $ax++; } } # Check for valid token token values - if (( $dbuser eq "" ) || ( $log_file eq "" ) || ( $dbhost eq "") || ( $dbpass eq "" ) ) { + if (( $dbuser eq "" ) || ( $log_file eq "" ) || ( $dbhost eq "") || ($dbpass eq "" ) ) { print "[ERROR] Bad Config values. Be sure that $archivo_cfg is a valid setup file"; exit; } # Open database to get days_purge days_compact values my $query; my $query_ready; my @data; my $rows_selected; - my $dbh = DBI->connect("DBI:mysql:pandora:$dbhost:3306",$dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 }); + my $dbh = DBI->connect("DBI:mysql:pandora:$dbhost:3306",$dbuser, $dbpass, {RaiseError => 1, AutoCommit => 1 }); $query = "select * from tconfig where token = 'days_purge'"; $query_ready = $dbh->prepare($query); $query_ready ->execute(); @@ -380,6 +386,101 @@ sub pandora_loadconfig { printf "Pandora DB now initialized and running (PURGE=$config_days_purge days, COMPACT=$config_days_compact days, STEP=$config_step_compact) ... \n\n"; } + +############################################################################### +## SUB pandora_checkdb_consistency (dbname, dbuser, dbpass, dbhost) +############################################################################### +sub pandora_checkdb_consistency { + + # 1. Check for modules that do not have tagente_estado but have tagente_module + + my $dbname = $_[0]; + my $dbuser = $_[1]; + my $dbpass = $_[2]; + my $dbhost = $_[3]; + my @query; + my $counter; + my $err; # error code in datecalc function + my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{RaiseError => 1, AutoCommit => 1 }); + + print "[CHECKDB] Checking database consistency (step1)... \n"; + my $query1 = "SELECT * FROM tagente_modulo"; + my $prep1 = $dbh->prepare($query1); + $prep1 ->execute; + my @datarow1; + if ($prep1->rows != 0) { + # for each record in tagente_modulo + while (@datarow1 = $prep1->fetchrow_array()) { + my $id_agente_modulo = $datarow1[0]; + # check if exist in tagente_estado and create if not + my $query2 = "SELECT * FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo"; + my $prep2 = $dbh->prepare($query2); + $prep2->execute; + # If have 0 items, we need to create tagente_estado record + if ($prep2->rows == 0) { + my $id_agente = $datarow1[1]; + my $query3 = "INSERT INTO tagente_estado (id_agente_modulo, datos, timestamp, cambio, estado, id_agente, last_try, utimestamp, current_interval, running_by, last_execution_try) VALUE ($id_agente_modulo, 0, '0000-00-00 00:00:00', 0, 100, $id_agente, '0000-00-00 00:00:00', 0, 0, 0, 0)"; + print "[CHECKDB] Inserting module $id_agente_modulo in state table \n"; + my $prep3 = $dbh->prepare($query3); + $prep3->execute; + $prep3->finish(); + } + $prep2->finish(); + } + } + $prep1->finish(); + + print "[CHECKDB] Checking database consistency (step2)... \n"; + # 2. Check for modules in tagente_estado that do not have tagente_modulo, if there is any, delete it + my $query1 = "SELECT * FROM tagente_estado"; + my $prep1 = $dbh->prepare($query1); + $prep1 ->execute; + my @datarow1; + if ($prep1->rows != 0) { + # for each record in tagente_modulo + while (@datarow1 = $prep1->fetchrow_array()) { + my $id_agente_modulo = $datarow1[1]; + # check if exist in tagente_estado and create if not + my $query2 = "SELECT * FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo"; + my $prep2 = $dbh->prepare($query2); + $prep2->execute; + # If have 0 items, we need to create tagente_estado record + if ($prep2->rows == 0) { + my $id_agente = $datarow1[1]; + my $query3 = "DELETE FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo"; + print "[CHECKDB] Deleting non-existing module $id_agente_modulo in state table \n"; + my $prep3 = $dbh->prepare($query3); + $prep3->execute; + $prep3->finish(); + } + $prep2->finish(); + } + } + $prep1->finish(); + + print "[CHECKDB] Deleting non-init data... \n"; + my $query4 = "SELECT * FROM tagente_estado WHERE utimestamp = 0"; + my $prep4 = $dbh->prepare($query4); + $prep4 ->execute; + my @datarow4; + if ($prep4->rows != 0) { + # for each record in tagente_modulo + while (@datarow4 = $prep4->fetchrow_array()) { + my $id_agente_modulo = $datarow4[1]; + my $query0 = "DELETE FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo"; + my $prep0 = $dbh->prepare($query0); + $prep0 ->execute; + $prep0->finish(); + } + } + $prep4->finish(); + + # Delete from tagente_estado + my $query0 = "DELETE FROM tagente_estado WHERE utimestamp = 0"; + my $prep0 = $dbh->prepare($query0); + $prep0 ->execute; + $prep0->finish(); +} ############################################################################## # SUB help_screen() @@ -394,11 +495,15 @@ sub help_screen{ exit; } -# ################################################################################# +# +############################################################################### # Program main begin -# ################################################################################# +# +############################################################################### sub pandoradb_main { pandora_purgedb ($config_days_purge, $dbname, $dbuser, $dbpass, $dbhost); pandora_compactdb ($config_days_compact, $dbname, $dbuser, $dbpass, $dbhost); + pandora_checkdb_consistency ($dbname, $dbuser, $dbpass, $dbhost); + print "\n"; exit; }