tyler.durden/pandorafms
1
0
Fork 0
mirror of https://github.com/pandorafms/pandorafms.git synced 2025-07-28 08:14:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

2007-08-28 Sancho Lerena <slerena@gmail.com>

Browse Source 
* util/pandora_db.pl: Added some fixes and new checks for database
        consistency. Also purge non-init modules (needed to avoid unused modules
        that take too much time on network servers!).



git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@627 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f
This commit is contained in:
slerena 2007-08-28 15:09:04 +00:00
parent 1f9f1ef933
commit 0d0d5a1c59
2 changed files with 195 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pandora_server/ChangeLog
View File

@ -1,3 +1,9 @@
2007-08-28 Sancho Lerena <slerena@gmail.com>
* util/pandora_db.pl: Added some fixes and new checks for database
consistency. Also purge non-init modules (needed to avoid unused modules
that take too much time on network servers!).
2007-08-27 Sancho Lerena <slerena@gmail.com> 2007-08-27 Sancho Lerena <slerena@gmail.com>
* lib/PandoraFMS/Config.pm: Updated version string. * lib/PandoraFMS/Config.pm: Updated version string.

161
pandora_server/util/pandora_db.pl
View File

@ -1,8 +1,8 @@
#!/usr/bin/perl #!/usr/bin/perl
################################################################################## ###############################################################################
# Pandora FMS DB Management # Pandora FMS DB Management
################################################################################ ###############################################################################
# Copyright (c) 2004-2006 Sancho Lerena, slerena@gmail.com # Copyright (c) 2004-2006 Sancho Lerena, slerena@gmail.com
# Copyright (c) 2005-2006 Artica Soluciones Tecnologicas S.L # Copyright (c) 2005-2006 Artica Soluciones Tecnologicas S.L
# #
@ -15,8 +15,8 @@
# GNU General Public License for more details. # GNU General Public License for more details.
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA
################################################################################ ###############################################################################
# Includes list # Includes list
use strict; use strict;
@ -25,7 +25,7 @@ use DBI; # DB interface with MySQL
use Date::Manip; # Date/Time manipulation use Date::Manip; # Date/Time manipulation
# version: define la version actual del programa # version: define la version actual del programa
my $version = "1.3 dev"; my $version = "1.3 PS070828";
# Setup variables # Setup variables
my $dirname=""; my $dirname="";
@ -52,15 +52,15 @@ pandora_loadconfig ($pandora_path);
# Begin pandora_server # Begin pandora_server
pandoradb_main(); pandoradb_main();
################################################################################## ###############################################################################
################################################################################## ###############################################################################
################################################################################## ###############################################################################
################################################################################## ###############################################################################
################################################################################## ###############################################################################
################################################################################## ###############################################################################
## SUB pandora_purgedb (days, dbname, dbuser, dbpass, dbhost) ## SUB pandora_purgedb (days, dbname, dbuser, dbpass, dbhost)
################################################################################## ###############################################################################
sub pandora_purgedb { sub pandora_purgedb {
# 1) Obtain last value for date limit # 1) Obtain last value for date limit
@ -84,6 +84,7 @@ sub pandora_purgedb {
my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S"); my $timestamp = &UnixDate("today","%Y-%m-%d %H:%M:%S");
my $limit_access = DateCalc("today","-24 hours",\$err); my $limit_access = DateCalc("today","-24 hours",\$err);
$limit_access = &UnixDate($limit_access,"%Y-%m-%d %H:%M:%S"); $limit_access = &UnixDate($limit_access,"%Y-%m-%d %H:%M:%S");
print "[PURGE] Deleting old data... \n";
# Lets insert the last value on $limit_timestamp + 1 minute for each id_agente_modulo # Lets insert the last value on $limit_timestamp + 1 minute for each id_agente_modulo
my $query_idag = "select count(distinct(id_agente_modulo)) from tagente_datos where timestamp < '$limit_timestamp'"; my $query_idag = "select count(distinct(id_agente_modulo)) from tagente_datos where timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag); my $idag = $dbh->prepare($query_idag);
@ -91,12 +92,12 @@ sub pandora_purgedb {
my @datarow; my @datarow;
@datarow = $idag->fetchrow_array(); @datarow = $idag->fetchrow_array();
if ($verbosity > 0){ if ($verbosity > 0){
print "Total different Modules delete: ".$datarow[0]."\n"; print "[PURGE] Total different Modules delete: ".$datarow[0]."\n";
} }
my $different_modules = $datarow[0]; my $different_modules = $datarow[0];
$idag->finish; $idag->finish;
my $query_idag = "select distinct(id_agente_modulo) from tagente_datos where timestamp < '$limit_timestamp'"; my $query_idag = "select distinct(id_agente_modulo) from tagente_datos WHERE timestamp < '$limit_timestamp'";
my $idag = $dbh->prepare($query_idag); my $idag = $dbh->prepare($query_idag);
$idag ->execute; $idag ->execute;
my @datarow; my @datarow;
@ -105,7 +106,7 @@ sub pandora_purgedb {
while (@datarow = $idag->fetchrow_array()) { while (@datarow = $idag->fetchrow_array()) {
$counter++; $counter++;
if ($verbosity > 0){ if ($verbosity > 0){
print "\r[PURGEDB] ".$counter / ($different_modules / 100)."% Deleted"; print "\r[PURGE] ".$counter / ($different_modules / 100)."% Deleted";
} }
$buffer = $datarow[0]; $buffer = $datarow[0];
my $query_idag2 = "select * from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer order by timestamp desc limit 1"; my $query_idag2 = "select * from tagente_datos where timestamp < '$limit_timestamp' and id_agente_modulo = $buffer order by timestamp desc limit 1";
@ -116,7 +117,7 @@ sub pandora_purgedb {
if ($idag2->rows != 0) { if ($idag2->rows != 0) {
while (@datarow2 = $idag2->fetchrow_array()) { while (@datarow2 = $idag2->fetchrow_array()) {
# Create Insert SQL for this data # Create Insert SQL for this data
$buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow2[2],'$limit_timestamp',$datarow2[4])"; $buffer3 = "insert into tagente_datos (id_agente_modulo,datos,timestamp,id_agente) values ($buffer,$datarow[2],'$limit_timestamp',$datarow2[4])";
} }
} }
# Execute DELETE # Execute DELETE
@ -135,7 +136,7 @@ sub pandora_purgedb {
} }
$idag->finish(); $idag->finish();
if ($verbosity > 0){ if ($verbosity > 0){
print "[PURGEDB] Deleting static data until $limit_timestamp \n"; print "[PURGE] Deleting static data until $limit_timestamp \n";
} }
$query[0] = "delete from tagente_datos_string where timestamp < '$limit_timestamp'"; $query[0] = "delete from tagente_datos_string where timestamp < '$limit_timestamp'";
$query[1] = "delete from tagent_access where timestamp < '$limit_access'"; $query[1] = "delete from tagent_access where timestamp < '$limit_access'";
@ -146,9 +147,9 @@ sub pandora_purgedb {
$dbh->disconnect(); $dbh->disconnect();
} }
################################################################################## ###############################################################################
## SUB pandora_compactdb (days, dbname, dbuser, dbpass, dbhost) ## SUB pandora_compactdb (days, dbname, dbuser, dbpass, dbhost)
################################################################################## ###############################################################################
sub pandora_compactdb { sub pandora_compactdb {
my $days = $_[0]; my $days = $_[0];
my $dbname = $_[1]; my $dbname = $_[1];
@ -176,13 +177,15 @@ sub pandora_compactdb {
# Get the first (oldest) timestamp in database to make it the marker for the end of query # Get the first (oldest) timestamp in database to make it the marker for the end of query
$query = "select min(timestamp) from tagente_datos "; $query = "SELECT min(timestamp) FROM tagente_datos ";
$query_ready = $dbh->prepare($query); $query_ready = $dbh->prepare($query);
$query_ready ->execute(); $query_ready ->execute();
@data_item = $query_ready->fetchrow_array(); @data_item = $query_ready->fetchrow_array();
$oldest_timestamp = @data_item[0]; $oldest_timestamp = @data_item[0];
$query_ready->finish; $query_ready->finish;
# If no data, skip this step
if ($oldest_timestamp != ""){
# We need to determine data ranges # We need to determine data ranges
# Calculate start limit for compactation, Today- X hour to older datetime # Calculate start limit for compactation, Today- X hour to older datetime
$limit_timestamp = DateCalc("today","-$days days",\$err); $limit_timestamp = DateCalc("today","-$days days",\$err);
@ -228,9 +231,9 @@ sub pandora_compactdb {
$value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-) $value = int($data_list{$key} / $data_list_items{$key}); # Media aritmetica :-)
$query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')"; $query="insert into tagente_datos (id_agente_modulo, datos, timestamp) values ($key, $value, '$limit_timestamp')";
$dbh->do($query); $dbh->do($query);
if ($verbosity > 0){ #if ($verbosity > 0){
print "DEBUG: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n"; # print "[DEBUG]: Datos para el id_agente_modulo # $key : Numero de datos ( $data_list_items{$key} ) valor total ( $data_list{$key} media ($value)) \n";
} #}
# Purge hash # Purge hash
delete $data_list{$key}; delete $data_list{$key};
delete $data_list_items{$key}; delete $data_list_items{$key};
@ -239,6 +242,9 @@ sub pandora_compactdb {
$limit_timestamp = $low_limit_timestamp; # To the next day !! $limit_timestamp = $low_limit_timestamp; # To the next day !!
$flag = Date_Cmp($oldest_timestamp,$limit_timestamp); $flag = Date_Cmp($oldest_timestamp,$limit_timestamp);
} until ($flag >= 0); } until ($flag >= 0);
} else {
print "[COMPACT] No data to pack ! \n";
}
$query_ready->finish(); $query_ready->finish();
$dbh->disconnect(); $dbh->disconnect();
} }
@ -249,9 +255,9 @@ sub pandora_compactdb {
############################################################################## ##############################################################################
sub pandora_init { sub pandora_init {
print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 ArticaST\n"; print "\nPandora FMS DB Tool $version Copyright (c) 2004-2007 Sancho Lerena\n";
print "This program is Free Software, licensed under the terms of GPL License v2\n"; print "This program is Free Software, licensed under the terms of GPL License v2\n";
print "You can download latest versions and documentation at http://pandora.sourceforge.net\n"; print "You can download latest versions and documentation at http://pandora.sf.net\n";
# Load config file from command line # Load config file from command line
if ($#ARGV == -1 ){ if ($#ARGV == -1 ){
@ -297,7 +303,7 @@ sub pandora_loadconfig {
open (CFG, "< $archivo_cfg"); open (CFG, "< $archivo_cfg");
while (<CFG>){ while (<CFG>){
$buffer_line = $_; $buffer_line = $_;
if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\!\*\@\#\%\$\~\"\']+)/){ if ($buffer_line =~ m/([\w-_\.]+)\s([0-9\w-_\.\/\?\&\=\)\(\_\-\\*\@\#\%\$\~\"\']+)/){
push @command_line,$1; push @command_line,$1;
push @command_line,$2; push @command_line,$2;
} }
@ -381,6 +387,101 @@ sub pandora_loadconfig {
printf "Pandora DB now initialized and running (PURGE=$config_days_purge days, COMPACT=$config_days_compact days, STEP=$config_step_compact) ... \n\n"; printf "Pandora DB now initialized and running (PURGE=$config_days_purge days, COMPACT=$config_days_compact days, STEP=$config_step_compact) ... \n\n";
} }
###############################################################################
## SUB pandora_checkdb_consistency (dbname, dbuser, dbpass, dbhost)
###############################################################################
sub pandora_checkdb_consistency {
# 1. Check for modules that do not have tagente_estado but have tagente_module
my $dbname = $_[0];
my $dbuser = $_[1];
my $dbpass = $_[2];
my $dbhost = $_[3];
my @query;
my $counter;
my $err; # error code in datecalc function
my $dbh = DBI->connect("DBI:mysql:$dbname:$dbhost:3306",$dbuser, $dbpass,{RaiseError => 1, AutoCommit => 1 });
print "[CHECKDB] Checking database consistency (step1)... \n";
my $query1 = "SELECT * FROM tagente_modulo";
my $prep1 = $dbh->prepare($query1);
$prep1 ->execute;
my @datarow1;
if ($prep1->rows != 0) {
# for each record in tagente_modulo
while (@datarow1 = $prep1->fetchrow_array()) {
my $id_agente_modulo = $datarow1[0];
# check if exist in tagente_estado and create if not
my $query2 = "SELECT * FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo";
my $prep2 = $dbh->prepare($query2);
$prep2->execute;
# If have 0 items, we need to create tagente_estado record
if ($prep2->rows == 0) {
my $id_agente = $datarow1[1];
my $query3 = "INSERT INTO tagente_estado (id_agente_modulo, datos, timestamp, cambio, estado, id_agente, last_try, utimestamp, current_interval, running_by, last_execution_try) VALUE ($id_agente_modulo, 0, '0000-00-00 00:00:00', 0, 100, $id_agente, '0000-00-00 00:00:00', 0, 0, 0, 0)";
print "[CHECKDB] Inserting module $id_agente_modulo in state table \n";
my $prep3 = $dbh->prepare($query3);
$prep3->execute;
$prep3->finish();
}
$prep2->finish();
}
}
$prep1->finish();
print "[CHECKDB] Checking database consistency (step2)... \n";
# 2. Check for modules in tagente_estado that do not have tagente_modulo, if there is any, delete it
my $query1 = "SELECT * FROM tagente_estado";
my $prep1 = $dbh->prepare($query1);
$prep1 ->execute;
my @datarow1;
if ($prep1->rows != 0) {
# for each record in tagente_modulo
while (@datarow1 = $prep1->fetchrow_array()) {
my $id_agente_modulo = $datarow1[1];
# check if exist in tagente_estado and create if not
my $query2 = "SELECT * FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo";
my $prep2 = $dbh->prepare($query2);
$prep2->execute;
# If have 0 items, we need to create tagente_estado record
if ($prep2->rows == 0) {
my $id_agente = $datarow1[1];
my $query3 = "DELETE FROM tagente_estado WHERE id_agente_modulo = $id_agente_modulo";
print "[CHECKDB] Deleting non-existing module $id_agente_modulo in state table \n";
my $prep3 = $dbh->prepare($query3);
$prep3->execute;
$prep3->finish();
}
$prep2->finish();
}
}
$prep1->finish();
print "[CHECKDB] Deleting non-init data... \n";
my $query4 = "SELECT * FROM tagente_estado WHERE utimestamp = 0";
my $prep4 = $dbh->prepare($query4);
$prep4 ->execute;
my @datarow4;
if ($prep4->rows != 0) {
# for each record in tagente_modulo
while (@datarow4 = $prep4->fetchrow_array()) {
my $id_agente_modulo = $datarow4[1];
my $query0 = "DELETE FROM tagente_modulo WHERE id_agente_modulo = $id_agente_modulo";
my $prep0 = $dbh->prepare($query0);
$prep0 ->execute;
$prep0->finish();
}
}
$prep4->finish();
# Delete from tagente_estado
my $query0 = "DELETE FROM tagente_estado WHERE utimestamp = 0";
my $prep0 = $dbh->prepare($query0);
$prep0 ->execute;
$prep0->finish();
}
############################################################################## ##############################################################################
# SUB help_screen() # SUB help_screen()
# Show a help screen an exits # Show a help screen an exits
@ -394,11 +495,15 @@ sub help_screen{
exit; exit;
} }
# ################################################################################# #
###############################################################################
# Program main begin # Program main begin
# ################################################################################# #
###############################################################################
sub pandoradb_main { sub pandoradb_main {
pandora_purgedb ($config_days_purge, $dbname, $dbuser, $dbpass, $dbhost); pandora_purgedb ($config_days_purge, $dbname, $dbuser, $dbpass, $dbhost);
pandora_compactdb ($config_days_compact, $dbname, $dbuser, $dbpass, $dbhost); pandora_compactdb ($config_days_compact, $dbname, $dbuser, $dbpass, $dbhost);
pandora_checkdb_consistency ($dbname, $dbuser, $dbpass, $dbhost);
print "\n";
exit; exit;
} }