From 5333dbb6e30e669c45ea4869419bd2f3802ce0d3 Mon Sep 17 00:00:00 2001 From: Calvo Date: Thu, 29 Jun 2023 17:06:59 +0200 Subject: [PATCH] WIP:Automonitorization Upgrades --- pandora_server/bin/pandora_server | 1 + pandora_server/lib/PandoraFMS/Core.pm | 212 +++++++++++++++++- pandora_server/lib/PandoraFMS/DB.pm | 15 ++ .../lib/PandoraFMS/ProducerConsumerServer.pm | 6 +- pandora_server/lib/PandoraFMS/Tools.pm | 80 +++++++ 5 files changed, 308 insertions(+), 6 deletions(-) diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index 8530b998c4..0de5d7e66a 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -551,6 +551,7 @@ sub pandora_server_tasks ($) { && !is_metaconsole($pa_config) && $counter % $pa_config->{'self_monitoring_interval'} == 0) { pandora_self_monitoring ($pa_config, $dbh); + pandora_installation_monitoring($pa_config, $dbh); } }; diff --git a/pandora_server/lib/PandoraFMS/Core.pm b/pandora_server/lib/PandoraFMS/Core.pm index 4fa53ee296..3a5ca2f513 100644 --- a/pandora_server/lib/PandoraFMS/Core.pm +++ b/pandora_server/lib/PandoraFMS/Core.pm @@ -102,6 +102,8 @@ Exported Functions: =item * C +=item * C + =back =head1 METHODS @@ -129,7 +131,7 @@ use Text::ParseWords; use Math::Trig; # Math functions # Debugging -#use Data::Dumper; +use Data::Dumper; # Force XML::Simple to use XML::Parser instead SAX to manage XML # due a bug processing some XML with blank spaces. @@ -262,6 +264,7 @@ our @EXPORT = qw( pandora_server_statistics pandora_self_monitoring pandora_thread_monitoring + pandora_installation_monitoring pandora_process_policy_queue pandora_sync_agents_integria pandora_get_integria_ticket_types @@ -6094,6 +6097,13 @@ sub pandora_self_monitoring ($$) { my $free_disk_spool = disk_free ($pa_config->{"incomingdir"}); $free_disk_spool = '' unless defined ($free_disk_spool); my $my_data_server = get_db_value ($dbh, "SELECT id_server FROM tserver WHERE server_type = ? AND name = '".$pa_config->{"servername"}."'", DATASERVER); + my $total_mem = total_mem(); + my $free_mem_percentage; + if(defined($total_mem) && $free_mem ne '') { + $free_mem_percentage = ($free_mem / $total_mem ) * 100; + } else { + $free_mem_percentage = ''; + } # Number of unknown agents my $agents_unknown = 0; @@ -6124,6 +6134,37 @@ sub pandora_self_monitoring ($$) { } my $elasticsearch_perfomance = enterprise_hook("elasticsearch_performance", [$pa_config, $dbh]); + my $num_threads = 0; + $num_threads = get_db_value ($dbh, "SELECT SUM(threads) FROM tserver WHERE name = '".$pa_config->{"servername"}."'"); + my $cpu_load = 0; + $cpu_load = cpu_load(); + + + ## Modules Networks average. + my $totalNetworkModules = get_db_value( + $dbh, + 'SELECT count(*) + FROM tagente_modulo + WHERE id_tipo_modulo + BETWEEN 6 AND 18' + ); + + my $totalModuleIntervalTime = get_db_value( + $dbh, + 'SELECT SUM(module_interval) + FROM tagente_modulo + WHERE id_tipo_modulo + BETWEEN 6 AND 18' + ); + + my $data_in_files = count_files_ext($pa_config->{"incomingdir"}, 'data'); + my $data_in_files_badxml = count_files_ext($pa_config->{"incomingdir"}, 'data_BADXML'); + my $averageTime = 0; + + if (defined($totalModuleIntervalTime) && defined($totalNetworkModules)) { + $averageTime = $totalNetworkModules / $totalModuleIntervalTime; + } + $xml_output .= $elasticsearch_perfomance if defined($elasticsearch_perfomance); @@ -6160,7 +6201,14 @@ sub pandora_self_monitoring ($$) { $xml_output .=" $free_mem"; $xml_output .=" "; } - + + $xml_output .=" "; + $xml_output .=" Free_RAM_perccentage"; + $xml_output .=" generic_data"; + $xml_output .=" $free_mem_percentage"; + $xml_output .=" %"; + $xml_output .=" "; + if (defined($free_disk_spool)) { $xml_output .=" "; $xml_output .=" FreeDisk_SpoolDir"; @@ -6169,6 +6217,43 @@ sub pandora_self_monitoring ($$) { $xml_output .=" "; } + $xml_output .=" "; + $xml_output .=" Total Threads"; + $xml_output .=" generic_data"; + $xml_output .=" $num_threads"; + $xml_output .=" "; + + $xml_output .=" "; + $xml_output .=" CPU Load"; + $xml_output .=" generic_data"; + $xml_output .=" $cpu_load"; + $xml_output .=" %"; + $xml_output .=" "; + + $xml_output .=" "; + $xml_output .=" Network Modules Int AVG"; + $xml_output .=" generic_data"; + $xml_output .=" $averageTime"; + $xml_output .=" seconds"; + $xml_output .=" "; + + if(defined($data_in_files)) { + $xml_output .=" "; + $xml_output .=" Data_in_files"; + $xml_output .=" generic_data"; + $xml_output .=" $data_in_files"; + $xml_output .=" "; + } + + if(defined($data_in_files_badxml)) { + $xml_output .=" "; + $xml_output .=" Data_in_BADXML_files"; + $xml_output .=" generic_data"; + $xml_output .=" $data_in_files_badxml"; + $xml_output .=" "; + } + + $xml_output .= ""; my $filename = $pa_config->{"incomingdir"}."/".$pa_config->{'servername'}.".self.".$utimestamp.".data"; @@ -6192,15 +6277,21 @@ sub pandora_thread_monitoring ($$$) { my $timestamp = strftime ("%Y-%m-%d %H:%M:%S", localtime()); my $xml_output = ""; - + my $module_parent = ""; + + # All trhead modules are "Status" module sons. + $module_parent = 'Status'; + $xml_output = "{'version'} . "' agent_name='".$pa_config->{'servername'} . "' agent_alias='".$pa_config->{'servername'} . "' interval='".$pa_config->{"self_monitoring_interval"}."' timestamp='".$timestamp."' >"; foreach my $server (@{$servers}) { - while (my ($tid, $stats) = each(%{$server->getProducerStats()})) { + my $producer_stats = $server->getProducerStats(); + while (my ($tid, $stats) = each(%{$producer_stats})) { $xml_output .=" "; $xml_output .=" " . uc($ServerTypes[$server->{'_server_type'}]) . " Producer Status"; $xml_output .=" generic_proc"; $xml_output .=" System"; $xml_output .=" " . (time() - $stats->{'tstamp'} < 2 * $pa_config->{"self_monitoring_interval"} ? 1 : 0) . ""; + $xml_output .=" " . $module_parent . ""; $xml_output .=" "; $xml_output .=" "; @@ -6209,6 +6300,16 @@ sub pandora_thread_monitoring ($$$) { $xml_output .=" Performance"; $xml_output .=" " . $stats->{'rate'} . ""; $xml_output .=" tasks/second"; + $xml_output .=" " . $module_parent . ""; + $xml_output .=" "; + + $xml_output .=" "; + $xml_output .=" " . uc($ServerTypes[$server->{'_server_type'}]) . " Producer Queued Elements"; + $xml_output .=" generic_data"; + $xml_output .=" Performance"; + $xml_output .=" " . ($#{$stats->{'task_queue'}} + 1) . ""; + $xml_output .=" tasks"; + $xml_output .=" " . $module_parent . ""; $xml_output .=" "; } @@ -6223,6 +6324,7 @@ sub pandora_thread_monitoring ($$$) { $xml_output .=" generic_proc"; $xml_output .=" System"; $xml_output .=" " . (time() - $stats->{'tstamp'} < 2 * $pa_config->{"self_monitoring_interval"} ? 1 : 0) . ""; + $xml_output .=" " . $module_parent . ""; $xml_output .=" "; $xml_output .=" "; @@ -6230,8 +6332,18 @@ sub pandora_thread_monitoring ($$$) { $xml_output .=" generic_data"; $xml_output .=" Performance"; $xml_output .=" " . $stats->{'rate'} . ""; + $xml_output .=" " . $module_parent . ""; $xml_output .=" tasks/second"; $xml_output .=" "; + + $xml_output .=" "; + $xml_output .=" " . uc($ServerTypes[$server->{'_server_type'}]) . " Producer Queued Elements"; + $xml_output .=" generic_data"; + $xml_output .=" Performance"; + $xml_output .=" " . ($#{$stats->{'task_queue'}} + 1) . ""; + $xml_output .=" tasks"; + $xml_output .=" " . $module_parent . ""; + $xml_output .=" "; } } $xml_output .= ""; @@ -6242,6 +6354,98 @@ sub pandora_thread_monitoring ($$$) { close (XMLFILE); } +########################################################################## +=head2 C<< pandora_installation_monitoring (I<$pa_config>, I<$dbh>, I<$servers>) >> + +Generate stats for Pandora FMS threads. + +=cut +########################################################################## +sub pandora_installation_monitoring($$) { + my ($pa_config, $dbh) = @_; + + my $utimestamp = time (); + my $timestamp = strftime ("%Y-%m-%d %H:%M:%S", localtime()); + my @modules; + + my $xml_output = ""; + $xml_output = "{'version'} . "' agent_name='pandora.internals' agent_alias='pandora.internals' interval='".$pa_config->{"self_monitoring_interval"}."' timestamp='".$timestamp."' >"; + + # Total amount of agents + my $module; + $module->{'name'} = "total_agents"; + $module->{'description'} = 'Total amount of agents'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(DISTINCT(id_agente)) FROM tagente'); + push(@modules, $module); + undef $module; + # Total amount of modules + $module->{'name'} = "total_modules"; + $module->{'description'} = 'Total modules'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(DISTINCT(id_agente_modulo)) FROM tagente_modulo'); + push(@modules, $module); + undef $module; + # Total groups + $module->{'name'} = "total_groups"; + $module->{'description'} = 'Total groups'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(DISTINCT(id_grupo)) FROM tgrupo'); + push(@modules, $module); + undef $module; + # Total module data records + $module->{'name'} = "total_data"; + $module->{'description'} = 'Total module data records'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(*) FROM tagente_datos'); + $module->{'interval'} = '86400'; + push(@modules, $module); + undef $module; + # Total module strimg data records + $module->{'name'} = "total_string_data"; + $module->{'description'} = 'Total module string data records'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(*) FROM tagente_datos_string'); + $module->{'interval'} = '86400'; + push(@modules, $module); + undef $module; + # Total agent access record + $module->{'name'} = "total_access_data"; + $module->{'description'} = 'Total agent access records'; + $module->{'data'} = get_db_value($dbh, 'SELECT COUNT(*) FROM tagent_access'); + push(@modules, $module); + undef $module; + + print Dumper(@modules); + foreach my $module_data (@modules) { + $xml_output .=" "; + $xml_output .=" " .$module_data->{'name'}. ""; + $xml_output .=" " . $module_data->{'data'} . ""; + + if(defined($module->{'description'})) { + $xml_output .=" " .$module_data->{'decription'}. ""; + } + if(defined($module->{'type'})) { + $xml_output .=" " .$module_data->{'type'}. ""; + } else { + $xml_output .=" generic_data"; + } + if(defined($module->{'unit'})) { + $xml_output .=" " .$module_data->{'unit'}. ""; + } + if(defined($module->{'module_parent'})) { + $xml_output .=" " .$module_data->{'module_parent'}. ""; + } + if(defined($module->{'interval'})) { + $xml_output .=" " .$module_data->{'interval'}. ""; + } + + $xml_output .=" "; + } + + $xml_output .= ""; + + my $filename = $pa_config->{"incomingdir"}."/pandora.internals.".$utimestamp.".data"; + open (XMLFILE, ">", $filename) or die "[FATAL] Could not write to the thread monitoring XML file '$filename'"; + print XMLFILE $xml_output; + close (XMLFILE); +} + ########################################################################## =head2 C<< set_master (I<$pa_config>, I<$dbh>) >> diff --git a/pandora_server/lib/PandoraFMS/DB.pm b/pandora_server/lib/PandoraFMS/DB.pm index c7544de2f7..0ece8d0c74 100644 --- a/pandora_server/lib/PandoraFMS/DB.pm +++ b/pandora_server/lib/PandoraFMS/DB.pm @@ -71,6 +71,7 @@ our @EXPORT = qw( get_agent_group get_agent_name get_agent_module_id + get_agent_module_id_by_name get_alert_template_module_id get_alert_template_name get_command_id @@ -698,6 +699,20 @@ sub get_agent_module_id ($$$) { return defined ($rc) ? $rc : -1; } +######################################################################## +## Return module id given the module name and agent name. +######################################################################## +sub get_agent_module_id_by_name ($$$) { + my ($dbh, $module_name, $agent_name) = @_; + + my $rc = get_db_value ( + $dbh, + 'SELECT id_agente_modulo + FROM tagente_modulo tam LEFT JOIN tagente ta ON tam.id_agente = ta.id_agente + WHERE tam.nombre = ? AND ta.nombre = ?', safe_input($module_name), $agent_name); + return defined ($rc) ? $rc : -1; +} + ########################################################################## ## Return template id given the template name. ########################################################################## diff --git a/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm b/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm index 770bc06522..51b0c3f35e 100644 --- a/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm +++ b/pandora_server/lib/PandoraFMS/ProducerConsumerServer.pm @@ -95,7 +95,8 @@ sub run ($$$$$) { 'tstamp' => time(), 'rate' => 0, 'rate_count' => 0, - 'rate_tstamp' => time() + 'rate_tstamp' => time(), + 'task_queue' => $task_queue, }); my $thr = threads->create ({'exit' => 'thread_only'}, @@ -127,7 +128,8 @@ sub run ($$$$$) { 'tstamp' => time(), 'rate' => 0, 'rate_count' => 0, - 'rate_tstamp' => time() + 'rate_tstamp' => time(), + 'task_queue' => $task_queue, }); # Launch producer thread diff --git a/pandora_server/lib/PandoraFMS/Tools.pm b/pandora_server/lib/PandoraFMS/Tools.pm index fb4b551a3a..8ffdba454b 100755 --- a/pandora_server/lib/PandoraFMS/Tools.pm +++ b/pandora_server/lib/PandoraFMS/Tools.pm @@ -140,6 +140,9 @@ our @EXPORT = qw( disk_free load_average free_mem + total_mem + cpu_load + count_files_ext md5 md5_init pandora_ping @@ -1405,6 +1408,83 @@ sub free_mem { return $free_mem; } +sub total_mem { + my $total_mem; + + my $OSNAME = $^O; + + if ($OSNAME eq "freebsd"){ + $total_mem = `/sbin/sysctl sysctl -b hw.physmem`; + # in kilobytes + $total_mem = $total_mem / 1024; + + } + elsif ($OSNAME eq "netbsd"){ + $total_mem = `cat /proc/meminfo | grep MemTotal | awk '{ print \$2 }'`; + } + elsif ($OSNAME eq "MSWin32"){ + $total_mem = `wmic ComputerSystem get TotalPhysicalMemory /Value`; + if ($total_mem =~ m/=(.*)$/gm) { + $total_mem = $1; + } else { + $total_mem = undef; + } + } + # by default LINUX calls + else { + $total_mem = `free | grep Mem | awk '{ print \$2 }'`; + } + return $total_mem; +} + + +################################################################################ +## SUB CPU load + # Get CPU load (%) +################################################################################ +sub cpu_load { + my $cpu_load; + + my $OSNAME = $^O; + + if ($OSNAME eq "MSWin32"){ + $cpu_load = `wmic cpu get loadpercentage|find /I /V "Loadpercentage" | findstr /r "[0-9]" `; + } + # by default LINUX calls + else { + $cpu_load = `top -bn 2 -d 0.01 | grep 'Cpu' | tail -n 1 | awk '{ print \$2+\$4+\$6 }'`; + } + + return $cpu_load; +} + +################################################################################ +## SUB count_files + # Count files in an specific folder by extension +################################################################################ +sub count_files_ext($$) { + my($path, $ext) = @_; + + my $count=0; + my $OSNAME = $^O; + + if ($OSNAME eq "MSWin32"){ + $path =~ '/^([a-zA-Z]:)?(\\\\[^\\/:*?\"<>|]+)*\\\\?/'; + my $drive = $1; + my $folder = $2; + + $count = `wmic datafile where "drive=\'$drive\' and path=\'$folder\' and extension=\'$ext\'" get /value | find /c "="`; + if ($count =~ m/=(.*)$/gm) { + $count = $1; + } + $count = undef; + + } else { + $count = `find $path -type f -name "*.$ext" | wc -l` + } + + return $count; +} ################################################################################ ## SUB ticks_totime # Transform a snmp timeticks count in a date