package PandoraFMS::DiscoveryServer; ################################################################################ # Pandora FMS Discovery Server. # Pandora FMS. the Flexible Monitoring System. http://www.pandorafms.org ################################################################################ # Copyright (c) 2005-2023 Pandora FMS # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License # as published by the Free Software Foundation; version 2 # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ################################################################################ use strict; use warnings; use threads; use threads::shared; use Thread::Semaphore; use IO::Socket::INET; use POSIX qw(strftime ceil); use JSON; use Encode qw(encode_utf8); use MIME::Base64; use File::Basename qw(dirname); use File::Copy; # Default lib dir for RPM and DEB packages BEGIN { push @INC, '/usr/lib/perl5'; } use PandoraFMS::Tools; use PandoraFMS::DB; use PandoraFMS::Core; use PandoraFMS::ProducerConsumerServer; use PandoraFMS::GIS; use PandoraFMS::Recon::Base; # Inherits from PandoraFMS::ProducerConsumerServer our @ISA = qw(PandoraFMS::ProducerConsumerServer); # Global variables my @TaskQueue :shared; my %PendingTasks :shared; my $Sem :shared; my $TaskSem :shared; # Some required constants, OS_X from tconfig_os. use constant { OS_OTHER => 10, OS_ROUTER => 17, OS_SWITCH => 18, STEP_SCANNING => 1, STEP_AFT => 2, STEP_TRACEROUTE => 3, STEP_GATEWAY => 4, STEP_MONITORING => 5, STEP_PROCESSING => 6, STEP_STATISTICS => 1, STEP_APP_SCAN => 2, STEP_CUSTOM_QUERIES => 3, DISCOVERY_REVIEW => 0, DISCOVERY_STANDARD => 1, DISCOVERY_RESULTS => 2, DISCOVERY_APP => 15, }; ################################################################################ # Discovery Server class constructor. ################################################################################ sub new ($$$$$$) { my ($class, $config, $dbh) = @_; return undef unless (defined($config->{'reconserver'}) && $config->{'reconserver'} == 1) || (defined($config->{'discoveryserver'}) && $config->{'discoveryserver'} == 1); if (! -e $config->{'nmap'}) { logger ($config, ' [E] ' . $config->{'nmap'} . " needed by " . $config->{'rb_product_name'} . " Discovery Server not found.", 1); print_message ($config, ' [E] ' . $config->{'nmap'} . " needed by " . $config->{'rb_product_name'} . " Discovery Server not found.", 1); return undef; } # Initialize semaphores and queues @TaskQueue = (); %PendingTasks = (); $Sem = Thread::Semaphore->new; $TaskSem = Thread::Semaphore->new (0); # Restart automatic recon tasks. db_do ($dbh, 'UPDATE trecon_task SET utimestamp = 0 WHERE id_recon_server = ? AND status <> -1 AND interval_sweep > 0', get_server_id ($dbh, $config->{'servername'}, DISCOVERYSERVER)); # Reset (but do not restart) manual recon tasks. db_do ($dbh, 'UPDATE trecon_task SET status = -1, summary = "cancelled" WHERE id_recon_server = ? AND status <> -1 AND interval_sweep = 0', get_server_id ($dbh, $config->{'servername'}, DISCOVERYSERVER)); # Call the constructor of the parent class my $self = $class->SUPER::new($config, DISCOVERYSERVER, \&PandoraFMS::DiscoveryServer::data_producer, \&PandoraFMS::DiscoveryServer::data_consumer, $dbh); bless $self, $class; return $self; } ################################################################################ # Run. ################################################################################ sub run ($) { my $self = shift; my $pa_config = $self->getConfig (); my $dbh = $self->getDBH(); print_message ($pa_config, " [*] Starting " . $pa_config->{'rb_product_name'} . " Discovery Server.", 1); my $threads = $pa_config->{'recon_threads'}; # Use hightest value if ($pa_config->{'discovery_threads'} > $pa_config->{'recon_threads'}) { $threads = $pa_config->{'discovery_threads'}; } $self->setNumThreads($threads); $self->SUPER::run (\@TaskQueue, \%PendingTasks, $Sem, $TaskSem); } ################################################################################ # Data producer. ################################################################################ sub data_producer ($) { my $self = shift; my ($pa_config, $dbh) = ($self->getConfig (), $self->getDBH ()); my @tasks; my $server_id = get_server_id ($dbh, $pa_config->{'servername'}, $self->getServerType ()); return @tasks unless defined ($server_id); # Manual tasks have interval_sweep = 0 # Manual tasks are "forced" like the other, setting the utimestamp to 1 # By default, after create a tasks it takes the utimestamp to 0 # Status -1 means "done". my @rows; if (pandora_is_master($pa_config) == 0) { @rows = get_db_rows ($dbh, 'SELECT * FROM trecon_task WHERE id_recon_server = ? AND disabled = 0 AND ((utimestamp = 0 AND interval_sweep != 0 OR status = 1) OR (status < 0 AND interval_sweep > 0 AND (utimestamp + interval_sweep) < UNIX_TIMESTAMP()))', $server_id); } else { @rows = get_db_rows ($dbh, 'SELECT * FROM trecon_task WHERE (id_recon_server = ? OR id_recon_server NOT IN (SELECT id_server FROM tserver WHERE status = 1 AND server_type = ?)) AND disabled = 0 AND ((utimestamp = 0 AND interval_sweep != 0 OR status = 1) OR (status < 0 AND interval_sweep > 0 AND (utimestamp + interval_sweep) < UNIX_TIMESTAMP()))', $server_id, DISCOVERYSERVER); } foreach my $row (@rows) { # Discovery apps must be fully set up. if ($row->{'type'} == DISCOVERY_APP && $row->{'setup_complete'} != 1) { logger($pa_config, 'Setup for recon app task ' . $row->{'id_app'} . ' not complete.', 10); next; } # Update task status update_recon_task ($dbh, $row->{'id_rt'}, 1); push (@tasks, $row->{'id_rt'}); } return @tasks; } ################################################################################ # Data consumer. ################################################################################ sub data_consumer ($$) { my ($self, $task_id) = @_; my ($pa_config, $dbh) = ($self->getConfig (), $self->getDBH ()); # Get server id. my $server_id = get_server_id($dbh, $pa_config->{'servername'}, $self->getServerType()); # Get recon task data my $task = get_db_single_row ($dbh, 'SELECT * FROM trecon_task WHERE id_rt = ?', $task_id); return -1 unless defined ($task); # Is it a recon script? if (defined ($task->{'id_recon_script'}) && ($task->{'id_recon_script'} != 0)) { exec_recon_script ($pa_config, $dbh, $task); return; } # Is it a discovery app? elsif ($task->{'type'} == DISCOVERY_APP) { exec_recon_app ($pa_config, $dbh, $task); return; } else { logger($pa_config, 'Starting recon task for net ' . $task->{'subnet'} . '.', 10); } eval { local $SIG{__DIE__}; my @subnets = split(/,/, safe_output($task->{'subnet'})); my @communities = split(/,/, safe_output($task->{'snmp_community'})); my @auth_strings = (); if(defined($task->{'auth_strings'})) { @auth_strings = split(/,/, safe_output($task->{'auth_strings'})); } my $main_event = pandora_event($pa_config, "[Discovery] Execution summary", $task->{'id_group'}, 0, 0, 0, 0, 'system', 0, $dbh ); my %cnf_extra; my $r = enterprise_hook( 'discovery_generate_extra_cnf', [ $pa_config, $dbh, $task, \%cnf_extra ] ); if (defined($r) && $r eq 'ERR') { # Could not generate extra cnf, skip this task. return; } if ($task->{'type'} == DISCOVERY_APP_SAP) { # SAP TASK, retrieve license. if (defined($task->{'field4'}) && $task->{'field4'} ne "") { $task->{'sap_license'} = $task->{'field4'}; } else { $task->{'sap_license'} = pandora_get_config_value( $dbh, 'sap_license' ); } # Retrieve credentials for task (optional). if (defined($task->{'auth_strings'}) && $task->{'auth_strings'} ne '' ) { my $key = credential_store_get_key( $pa_config, $dbh, $task->{'auth_strings'} ); # Inside an eval, here it shouln't fail unless bad configured. $task->{'username'} = $key->{'username'}; $task->{'password'} = $key->{'password'}; } } if (!is_empty($task->{'recon_ports'})) { # Accept only valid symbols. if ($task->{'recon_ports'} !~ /[\d\-\,\ ]+/) { $task->{'recon_ports'} = ''; } } my $recon = new PandoraFMS::Recon::Base( parent => $self, communities => \@communities, dbh => $dbh, group_id => $task->{'id_group'}, id_os => $task->{'id_os'}, id_network_profile => $task->{'id_network_profile'}, os_detection => $task->{'os_detect'}, parent_detection => $task->{'parent_detection'}, parent_recursion => $task->{'parent_recursion'}, pa_config => $pa_config, recon_ports => $task->{'recon_ports'}, resolve_names => $task->{'resolve_names'}, snmp_auth_user => $task->{'snmp_auth_user'}, snmp_auth_pass => $task->{'snmp_auth_pass'}, snmp_auth_method => $task->{'snmp_auth_method'}, snmp_checks => $task->{'snmp_checks'}, snmp_enabled => $task->{'snmp_enabled'}, snmp_privacy_method => $task->{'snmp_privacy_method'}, snmp_privacy_pass => $task->{'snmp_privacy_pass'}, snmp_security_level => $task->{'snmp_security_level'}, snmp_timeout => $task->{'snmp_timeout'}, snmp_version => $task->{'snmp_version'}, snmp_skip_non_enabled_ifs => $task->{'snmp_skip_non_enabled_ifs'}, subnets => \@subnets, task_id => $task->{'id_rt'}, vlan_cache_enabled => $task->{'vlan_enabled'}, wmi_enabled => $task->{'wmi_enabled'}, rcmd_enabled => $task->{'rcmd_enabled'}, rcmd_timeout => $pa_config->{'rcmd_timeout'}, rcmd_timeout_bin => $pa_config->{'rcmd_timeout_bin'}, auth_strings_array => \@auth_strings, autoconfiguration_enabled => $task->{'autoconfiguration_enabled'}, main_event_id => $main_event, server_id => $server_id, %{$pa_config}, task_data => $task, public_url => PandoraFMS::Config::pandora_get_tconfig_token($dbh, 'public_url', ''), %cnf_extra ); $recon->scan(); # Clean tmp file. if (defined($cnf_extra{'creds_file'}) && -f $cnf_extra{'creds_file'}) { unlink($cnf_extra{'creds_file'}); } # Clean one shot tasks if ($task->{'type'} eq DISCOVERY_DEPLOY_AGENTS) { db_delete_limit($dbh, ' trecon_task ', ' id_rt = ? ', 1, $task->{'id_rt'}); } }; if ($@) { logger( $pa_config, 'Cannot execute Discovery task: ' . safe_output($task->{'name'}) . $@, 10 ); update_recon_task ($dbh, $task_id, -1); return; } } ################################################################################ # Update recon task status. ################################################################################ sub update_recon_task ($$$) { my ($dbh, $id_task, $status) = @_; db_do ($dbh, 'UPDATE trecon_task SET utimestamp = ?, status = ? WHERE id_rt = ?', time (), $status, $id_task); } ################################################################################ # Executes recon scripts ################################################################################ sub exec_recon_script ($$$) { my ($pa_config, $dbh, $task) = @_; # Get recon plugin data my $script = get_db_single_row ($dbh, 'SELECT * FROM trecon_script WHERE id_recon_script = ?', $task->{'id_recon_script'}); return -1 unless defined ($script); logger($pa_config, 'Executing recon script ' . safe_output($script->{'name'}), 10); my $command = safe_output($script->{'script'}); my $macros = safe_output($task->{'macros'}); # \r and \n should be escaped for p_decode_json(). $macros =~ s/\n/\\n/g; $macros =~ s/\r/\\r/g; my $decoded_macros; if ($macros) { eval { $decoded_macros = p_decode_json($pa_config, $macros); }; } my $macros_parameters = ''; # Add module macros as parameter if(ref($decoded_macros) eq "HASH") { # Convert the hash to a sorted array my @sorted_macros; while (my ($i, $m) = each (%{$decoded_macros})) { $sorted_macros[$i] = $m; } # Remove the 0 position shift @sorted_macros; foreach my $m (@sorted_macros) { $macros_parameters = $macros_parameters . ' "' . $m->{"value"} . '"'; } } my $ent_script = 0; my $args = enterprise_hook( 'discovery_custom_recon_scripts', [$pa_config, $dbh, $task, $script] ); if (!$args) { $args = '"'.$task->{'id_rt'}.'" '; $args .= '"'.$task->{'id_group'}.'" '; $args .= $macros_parameters; } else { $ent_script = 1; } if (-x $command) { my $exec_output = `$command $args 2>&1`; log_execution($pa_config, $task->{'id_rt'}, "$command $args", $exec_output); logger($pa_config, "Execution output: \n". $exec_output, 10); } else { logger($pa_config, "Cannot execute recon task command $command.", 10); } # Only update the timestamp in case something went wrong. The script should set the status. db_do ($dbh, 'UPDATE trecon_task SET utimestamp = ? WHERE id_rt = ?', time (), $task->{'id_rt'}); if ($ent_script == 1) { enterprise_hook('discovery_clean_custom_recon',[$pa_config, $dbh, $task, $script]); } logger($pa_config, 'Done executing recon script ' . safe_output($script->{'name'}), 10); return 0; } ################################################################################ # Executes recon apps. ################################################################################ sub exec_recon_app ($$$) { my ($pa_config, $dbh, $task) = @_; # Get execution, macro and script data. my @executions = get_db_rows($dbh, 'SELECT * FROM tdiscovery_apps_executions WHERE id_app = ?', $task->{'id_app'}); my @scripts = get_db_rows($dbh, 'SELECT * FROM tdiscovery_apps_scripts WHERE id_app = ?', $task->{'id_app'}); logger($pa_config, 'Executing recon app ID ' . $task->{'id_app'}, 10); # Configure macros. my %macros = ( "__taskMD5__" => md5($task->{'id_rt'}), "__taskInterval__" => $task->{'interval_sweep'}, "__taskGroup__" => get_group_name($dbh, $task->{'id_group'}), "__taskGroupID__" => $task->{'id_group'}, "__temp__" => $pa_config->{'temporal'}, "__incomingDir__" => $pa_config->{'incomingdir'}, "__consoleAPIURL__" => $pa_config->{'console_api_url'}, "__consoleAPIPass__" => $pa_config->{'console_api_pass'}, "__consoleUser__" => $pa_config->{'console_user'}, "__consolePass__" => $pa_config->{'console_pass'}, "__pandoraServerConf__" => $pa_config->{'pandora_path'}, get_recon_app_macros($pa_config, $dbh, $task), get_recon_script_macros($pa_config, $dbh, $task) ); # Dump macros to disk. dump_recon_app_macros($pa_config, $dbh, $task, \%macros); # Run executions. my $status = -1; my @summary; for (my $i = 0; $i < scalar(@executions); $i++) { my $execution = $executions[$i]; # NOTE: Add the redirection before calling subst_alert_macros to prevent it from escaping quotes. my $cmd = $pa_config->{'plugin_exec'} . ' ' . $task->{'executions_timeout'} . ' ' . subst_alert_macros(safe_output($execution->{'execution'}) . ' 2>&1', \%macros); logger($pa_config, 'Executing command for recon app ID ' . $task->{'id_app'} . ': ' . $cmd, 10); my $output_json = `$cmd`; # Something went wrong. my $rc = $? >> 8; if ($rc != 0) { $status = -2; } # Timeout specific mesage. if ($rc == 124) { push(@summary, "The execution timed out."); next; } # No output message. if (!defined($output_json)) { push(@summary, "The execution returned no output. Is the server out of memory?"); next; } # Parse the output. my $output = eval { local $SIG{'__DIE__'}; decode_json($output_json); }; # Non-JSON output. if (!defined($output)) { push(@summary, $output_json); next; } # Process monitoring data. if (ref($output) eq 'HASH' && defined($output->{'monitoring_data'})) { my $recon = new PandoraFMS::Recon::Base( dbh => $dbh, group_id => $task->{'id_group'}, id_os => $task->{'id_os'}, pa_config => $pa_config, snmp_enabled => 0, task_id => $task->{'id_rt'}, task_data => $task, ); $recon->create_agents($output->{'monitoring_data'}); delete($output->{'monitoring_data'}); } # Store output data. push(@summary, $output); # Update the progress. update_recon_task($dbh, $task->{'id_rt'}, int((100 * ($i + 1)) / scalar(@executions))); } # Parse the output. my $summary_json = eval { local $SIG{'__DIE__'}; encode_json(\@summary); }; if (!defined($summary_json)) { logger($pa_config, 'Invalid summary for recon app ID ' . $task->{'id_app'}, 10); } else { db_do($dbh, "UPDATE trecon_task SET summary=? WHERE id_rt=?", $summary_json, $task->{'id_rt'}); pandora_audit ($pa_config, 'Discovery task' . ' Executed task '.$task->{'name'}.'#'.$task->{'id_app'}, 'SYSTEM', 'Discovery task', $dbh); } update_recon_task($dbh, $task->{'id_rt'}, $status); return; } ################################################################################ # Processe app macros and return them ready to be used by subst_alert_macros. ################################################################################ sub get_recon_app_macros ($$$) { my ($pa_config, $dbh, $task) = @_; my %macros; # Get a list of macros for the given task. my @macro_array = get_db_rows($dbh, 'SELECT * FROM tdiscovery_apps_tasks_macros WHERE id_task = ?', $task->{'id_rt'}); foreach my $macro_item (@macro_array) { my $macro_id = safe_output($macro_item->{'id_task'}); my $macro_name = safe_output($macro_item->{'macro'}); my $macro_type = $macro_item->{'type'}; my $macro_value = safe_output($macro_item->{'value'}); my $computed_value = ''; # The value can be a JSON array of values. my $value_array = eval { local $SIG{'__DIE__'}; decode_json($macro_value); }; if (defined($value_array) && ref($value_array) eq 'ARRAY') { # Multi value macro. my @tmp; foreach my $value_item (@{$value_array}) { push(@tmp, get_recon_macro_value($pa_config, $dbh, $macro_type, $value_item)); } $computed_value = p_encode_json($pa_config, \@tmp); if (!defined($computed_value)) { logger($pa_config, "Error encoding macro $macro_name for task ID " . $task->{'id_rt'}, 10); next; } } else { # Single value macro. $computed_value = get_recon_macro_value($pa_config, $dbh, $macro_type, $macro_value); } # Store the computed value. $macros{$macro_name} = $computed_value; } return %macros; } ################################################################################ # Dump macros that must be saved to disk. # The macros dictionary is modified in-place. ################################################################################ sub dump_recon_app_macros ($$$$) { my ($pa_config, $dbh, $task, $macros) = @_; # Get a list of macros that must be dumped to disk. my @macro_array = get_db_rows($dbh, 'SELECT * FROM tdiscovery_apps_tasks_macros WHERE id_task = ? AND temp_conf = 1', $task->{'id_rt'}); foreach my $macro_item (@macro_array) { # Make sure the macro has already been parsed. my $macro_name = safe_output($macro_item->{'macro'}); next unless defined($macros->{$macro_name}); my $macro_value = $macros->{$macro_name}; my $macro_id = safe_output($macro_item->{'id_task'}); # Save the computed value to a temporary file. my $temp_dir = $pa_config->{'incomingdir'} . '/discovery/tmp'; mkdir($temp_dir) if (! -d $temp_dir); my $fname = $temp_dir . '/' . md5($task->{'id_rt'} . '_' . $macro_name) . '.macro'; eval { open(my $fh, '>', $fname) or die($!); print $fh subst_alert_macros($macro_value, $macros); close($fh); }; if ($@) { logger($pa_config, "Error writing macro $macro_name for task ID " . $task->{'id_rt'} . " to disk: $@", 10); next; } # Set the macro value to the temporary file name. $macros->{$macro_name} = $fname; } } ################################################################################ # Processe recon script macros and return them ready to be used by # subst_alert_macros. ################################################################################ sub get_recon_script_macros ($$$) { my ($pa_config, $dbh, $task) = @_; my %macros; # Get a list of script macros. my @macro_array = get_db_rows($dbh, 'SELECT * FROM tdiscovery_apps_scripts WHERE id_app = ?', $task->{'id_app'}); foreach my $macro_item (@macro_array) { my $macro_name = safe_output($macro_item->{'macro'}); my $macro_value = safe_output($macro_item->{'value'}); # Compose the full path to the script: /discovery//