diff --git a/pandora_server/ChangeLog b/pandora_server/ChangeLog index 4c4889b951..8aebee4b64 100644 --- a/pandora_server/ChangeLog +++ b/pandora_server/ChangeLog @@ -1,3 +1,11 @@ +2013-04-24 Sancho Lerena + + * util/pandora_db.pl: patched the code of tagente_datos purge to have n chunks, + and in each chunk run a DELETE of only 1000 records at a time (with a LIMIT), + adding a 0,0001 second usleep() there, gives time to MySQL to put other + operations between, and locks doesn't appear to be a problem anymore. This + should solve many problems detected in the past. + 2013-04-21 Junichi Satoh * FreeBSD/pandora_server.conf, NetBSD/pandora_server.conf: Updated diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index c5298eb0b0..af0a73086c 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -38,9 +38,10 @@ my $version = "5.0dev PS130423"; my %conf; my $BIG_OPERATION_STEP = 100; # Long operations are divided in XX steps for performance - -# FLUSH in each IO -$| = 0; +my $SMALL_OPERATION_STEP = 1000; # Each long operations has a LIMIT of SMALL_OPERATION_STEP to avoid locks. + #Increate to 3000~5000 in fast systems decrease to 500 or 250 on systems with locks +# FLUSH in each IO +$| = 1; # Init pandora_init(\%conf); @@ -89,28 +90,45 @@ sub pandora_purgedb ($$) { my $first_mark; my $total_time; my $purge_steps; + my $purge_count; - # Numeric data deletion + my $mark1; + my $mark2; + + # Numeric data deletion print "[PURGE] Deleting old data... \n"; - + # This could be very timing consuming, so make this operation in $BIG_OPERATION_STEP # steps (100 fixed by default) # Starting from the oldest record on the table - + + # WARNING. This code is EXTREMELLY important. This block (data deletion) could KILL a database if + # you alter code and you don't know exactly what are you doing. Please take in mind this code executes each hour + # and has been patches MANY times. Before altering anything, think twice ! + $first_mark = get_db_value ($dbh, 'SELECT utimestamp FROM tagente_datos ORDER BY utimestamp ASC LIMIT 1'); if (defined ($first_mark)) { $total_time = $ulimit_timestamp - $first_mark; $purge_steps = int($total_time / $BIG_OPERATION_STEP); - + for (my $ax = 1; $ax <= $BIG_OPERATION_STEP; $ax++){ - db_do ($dbh, "DELETE FROM tagente_datos WHERE utimestamp < ". ($first_mark + ($purge_steps * $ax)) . " AND utimestamp >= ". $first_mark ); - print "[PURGE] Data deletion Progress %$ax .. \r"; - # Do a nanosleep here for 0,01 sec - usleep (10000); + $mark1 = $first_mark + ($purge_steps * $ax); + $mark2 = $first_mark + ($purge_steps * ($ax -1)); + + # Let's split the intervals in $SMALL_OPERATION_STEP deletes each + $purge_count = get_db_value ($dbh, "SELECT COUNT(id_agente_modulo) FROM tagente_datos WHERE utimestamp < $mark1 AND utimestamp > $mark2"); + while ($purge_count > 0){ + print "."; + db_do ($dbh, "DELETE FROM tagente_datos WHERE utimestamp < $mark1 AND utimestamp > $mark2 LIMIT $SMALL_OPERATION_STEP"); + # Do a nanosleep here for 0,001 sec + usleep (10000); + $purge_count = $purge_count - $SMALL_OPERATION_STEP; + } + print "\n[PURGE] Data deletion Progress (".$ax."%) "; } - print "\n"; + print "\n"; } else { print "[PURGE] No data in tagente_datos\n"; } @@ -593,13 +611,11 @@ sub pandora_checkdb_integrity { # Delete all records on tgroup_stats db_do ($dbh, 'DELETE FROM tgroup_stat'); - print "[INTEGRITY] Deleting non-used IP addresses \n"; - + + #print "[INTEGRITY] Deleting non-used IP addresses \n"; + # DISABLED - Takes too much time and benefits of this are unclear.... # Delete all non-used IP addresses from taddress - db_do ($dbh, 'DELETE FROM taddress WHERE id_a NOT IN (SELECT id_a FROM taddress_agent)'); - - # Do a nanosleep here for 0,01 sec - usleep (10000); + #db_do ($dbh, 'DELETE FROM taddress WHERE id_a NOT IN (SELECT id_a FROM taddress_agent)'); print "[INTEGRITY] Deleting orphan alerts \n"; @@ -616,7 +632,7 @@ sub pandora_checkdb_integrity { # Delete orphan data_inc reference records db_do ($dbh, 'DELETE FROM tagente_datos_inc WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); - + # Check enterprise tables enterprise_hook ('pandora_checkdb_integrity_enterprise', [$dbh]); }