tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

2013-04-24 Sancho Lerena <slerena@artica.es> 

* util/pandora_db.pl: patched the code of tagente_datos purge to have n chunks, 
        and in each chunk run a DELETE of only 1000 records at a time (with a LIMIT), 
        adding a 0,0001 second usleep() there, gives time to MySQL to put other 
        operations between, and locks doesn't appear to be a problem anymore. This
        should solve many problems detected in the past.




git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@8040 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f
This commit is contained in:
slerena 2013-04-24 01:12:50 +00:00
parent 8238ca89d7
commit a99f0aba2c
2 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pandora_server/ChangeLog
View File

@ -1,3 +1,11 @@
2013-04-24 Sancho Lerena <slerena@artica.es>
* util/pandora_db.pl: patched the code of tagente_datos purge to have n chunks,
and in each chunk run a DELETE of only 1000 records at a time (with a LIMIT),
adding a 0,0001 second usleep() there, gives time to MySQL to put other
operations between, and locks doesn't appear to be a problem anymore. This
should solve many problems detected in the past.
2013-04-21 Junichi Satoh <junichi@rworks.jp> 2013-04-21 Junichi Satoh <junichi@rworks.jp>
* FreeBSD/pandora_server.conf, NetBSD/pandora_server.conf: Updated * FreeBSD/pandora_server.conf, NetBSD/pandora_server.conf: Updated

54
pandora_server/util/pandora_db.pl
View File

@ -38,9 +38,10 @@ my $version = "5.0dev PS130423";
my %conf; my %conf;
my $BIG_OPERATION_STEP = 100; # Long operations are divided in XX steps for performance my $BIG_OPERATION_STEP = 100; # Long operations are divided in XX steps for performance
my $SMALL_OPERATION_STEP = 1000; # Each long operations has a LIMIT of SMALL_OPERATION_STEP to avoid locks.
# FLUSH in each IO #Increate to 3000~5000 in fast systems decrease to 500 or 250 on systems with locks
$| = 0; # FLUSH in each IO
$| = 1;
# Init # Init
pandora_init(\%conf); pandora_init(\%conf);
@ -89,28 +90,45 @@ sub pandora_purgedb ($$) {
my $first_mark; my $first_mark;
my $total_time; my $total_time;
my $purge_steps; my $purge_steps;
my $purge_count;
# Numeric data deletion my $mark1;
my $mark2;
# Numeric data deletion
print "[PURGE] Deleting old data... \n"; print "[PURGE] Deleting old data... \n";
# This could be very timing consuming, so make this operation in $BIG_OPERATION_STEP # This could be very timing consuming, so make this operation in $BIG_OPERATION_STEP
# steps (100 fixed by default) # steps (100 fixed by default)
# Starting from the oldest record on the table # Starting from the oldest record on the table
# WARNING. This code is EXTREMELLY important. This block (data deletion) could KILL a database if
# you alter code and you don't know exactly what are you doing. Please take in mind this code executes each hour
# and has been patches MANY times. Before altering anything, think twice !
$first_mark = get_db_value ($dbh, 'SELECT utimestamp FROM tagente_datos ORDER BY utimestamp ASC LIMIT 1'); $first_mark = get_db_value ($dbh, 'SELECT utimestamp FROM tagente_datos ORDER BY utimestamp ASC LIMIT 1');
if (defined ($first_mark)) { if (defined ($first_mark)) {
$total_time = $ulimit_timestamp - $first_mark; $total_time = $ulimit_timestamp - $first_mark;
$purge_steps = int($total_time / $BIG_OPERATION_STEP); $purge_steps = int($total_time / $BIG_OPERATION_STEP);
for (my $ax = 1; $ax <= $BIG_OPERATION_STEP; $ax++){ for (my $ax = 1; $ax <= $BIG_OPERATION_STEP; $ax++){
db_do ($dbh, "DELETE FROM tagente_datos WHERE utimestamp < ". ($first_mark + ($purge_steps * $ax)) . " AND utimestamp >= ". $first_mark );
print "[PURGE] Data deletion Progress %$ax .. \r";
# Do a nanosleep here for 0,01 sec $mark1 = $first_mark + ($purge_steps * $ax);
usleep (10000); $mark2 = $first_mark + ($purge_steps * ($ax -1));
# Let's split the intervals in $SMALL_OPERATION_STEP deletes each
$purge_count = get_db_value ($dbh, "SELECT COUNT(id_agente_modulo) FROM tagente_datos WHERE utimestamp < $mark1 AND utimestamp > $mark2");
while ($purge_count > 0){
print ".";
db_do ($dbh, "DELETE FROM tagente_datos WHERE utimestamp < $mark1 AND utimestamp > $mark2 LIMIT $SMALL_OPERATION_STEP");
# Do a nanosleep here for 0,001 sec
usleep (10000);
$purge_count = $purge_count - $SMALL_OPERATION_STEP;
}
print "\n[PURGE] Data deletion Progress (".$ax."%) ";
} }
print "\n"; print "\n";
} else { } else {
print "[PURGE] No data in tagente_datos\n"; print "[PURGE] No data in tagente_datos\n";
} }
@ -593,13 +611,11 @@ sub pandora_checkdb_integrity {
# Delete all records on tgroup_stats # Delete all records on tgroup_stats
db_do ($dbh, 'DELETE FROM tgroup_stat'); db_do ($dbh, 'DELETE FROM tgroup_stat');
print "[INTEGRITY] Deleting non-used IP addresses \n";
#print "[INTEGRITY] Deleting non-used IP addresses \n";
# DISABLED - Takes too much time and benefits of this are unclear....
# Delete all non-used IP addresses from taddress # Delete all non-used IP addresses from taddress
db_do ($dbh, 'DELETE FROM taddress WHERE id_a NOT IN (SELECT id_a FROM taddress_agent)'); #db_do ($dbh, 'DELETE FROM taddress WHERE id_a NOT IN (SELECT id_a FROM taddress_agent)');
# Do a nanosleep here for 0,01 sec
usleep (10000);
print "[INTEGRITY] Deleting orphan alerts \n"; print "[INTEGRITY] Deleting orphan alerts \n";
@ -616,7 +632,7 @@ sub pandora_checkdb_integrity {
# Delete orphan data_inc reference records # Delete orphan data_inc reference records
db_do ($dbh, 'DELETE FROM tagente_datos_inc WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)'); db_do ($dbh, 'DELETE FROM tagente_datos_inc WHERE id_agente_modulo NOT IN (SELECT id_agente_modulo FROM tagente_modulo)');
# Check enterprise tables # Check enterprise tables
enterprise_hook ('pandora_checkdb_integrity_enterprise', [$dbh]); enterprise_hook ('pandora_checkdb_integrity_enterprise', [$dbh]);
} }