diff --git a/pandora_console/extras/mr/62.sql b/pandora_console/extras/mr/62.sql index 6632e39f2b..eaef394fcc 100644 --- a/pandora_console/extras/mr/62.sql +++ b/pandora_console/extras/mr/62.sql @@ -1,5 +1,14 @@ START TRANSACTION; +ALTER TABLE `tdatabase` ADD COLUMN `ssh_status` TINYINT UNSIGNED DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `db_status` TINYINT UNSIGNED DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `replication_status` TINYINT UNSIGNED DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `replication_delay` BIGINT DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `master` TINYINT UNSIGNED DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `utimestamp` BIGINT DEFAULT 0; +ALTER TABLE `tdatabase` ADD COLUMN `mysql_version` VARCHAR(10) DEFAULT ''; +ALTER TABLE `tdatabase` ADD COLUMN `pandora_version` VARCHAR(10) DEFAULT ''; + UPDATE tconfig_os SET `icon_name` = 'linux@os.svg' WHERE `id_os` = 1; UPDATE tconfig_os SET `icon_name` = 'solaris@os.svg' WHERE `id_os` = 2; UPDATE tconfig_os SET `icon_name` = 'aix@os.svg' WHERE `id_os` = 3; @@ -166,4 +175,6 @@ CREATE TABLE IF NOT EXISTS `tfavmenu_user` ( `section` VARCHAR(255) NOT NULL, PRIMARY KEY (`id`)); +INSERT INTO `tconfig` (`token`, `value`) VALUES ('legacy_database_ha', 1); + COMMIT; diff --git a/pandora_console/include/config_process.php b/pandora_console/include/config_process.php index ec90b4d8c2..4375ffa121 100644 --- a/pandora_console/include/config_process.php +++ b/pandora_console/include/config_process.php @@ -153,6 +153,19 @@ if (! defined('ENTERPRISE_DIR')) { } db_select_engine(); + +if (empty($config['remote_config']) === false + && file_exists($config['remote_config'].'/conf/'.PANDORA_HA_FILE) + && filesize($config['remote_config'].'/conf/'.PANDORA_HA_FILE) > 0 +) { + $data = file_get_contents($config['remote_config'].'/conf/'.PANDORA_HA_FILE); + if (empty($data) === false) { + $ip_list = explode(',', $data); + // Connects to the first pandora_ha_dbs.conf database. + $config['dbhost'] = trim($ip_list[0]); + } +} + $config['dbconnection'] = db_connect(); require_once $ownDir.'functions_config.php'; diff --git a/pandora_console/include/constants.php b/pandora_console/include/constants.php index dc9fcd0b4d..6b86c50c6d 100644 --- a/pandora_console/include/constants.php +++ b/pandora_console/include/constants.php @@ -1,4 +1,5 @@ {"unknown_block_size"} = 1000; # 7.0.769 + $pa_config->{"ha_mode"} = "pacemaker"; # 7.0.770 + $pa_config->{"ha_file"} = undef; # 7.0.770 + $pa_config->{"ha_hosts_file"} = '/var/spool/pandora/data_in/conf/pandora_ha_hosts.conf'; # 7.0.770 + $pa_config->{"ha_connect_retries"} = 2; # 7.0.770 + $pa_config->{"ha_connect_delay"} = 1; # 7.0.770 + $pa_config->{"ha_dbuser"} = undef; # 7.0.770 + $pa_config->{"ha_dbpass"} = undef; # 7.0.770 + $pa_config->{"ha_hosts"} = undef; # 7.0.770 + $pa_config->{"ha_resync"} = '/usr/share/pandora_server/util/pandora_ha_resync_slave.sh'; # 7.0.770 + $pa_config->{"ha_resync_log"} = '/var/log/pandora/pandora_ha_resync.log'; # 7.0.770 + $pa_config->{"ha_sshuser"} = 'pandora'; # 7.0.770 + $pa_config->{"ha_sshport"} = 22; # 7.0.770 + + $pa_config->{"ha_max_splitbrain_retries"} = 2; + $pa_config->{"ha_resync_sleep"} = 10; + + $pa_config->{"repl_dbuser"} = undef; # 7.0.770 + $pa_config->{"repl_dbpass"} = undef; # 7.0.770 + # Check for UID0 if ($pa_config->{"quiet"} != 0){ if ($> == 0){ @@ -1283,9 +1302,36 @@ sub pandora_load_config { } # Pandora HA extra + elsif ($parametro =~ m/^ha_mode\s(.*)/i) { + $pa_config->{'ha_mode'} = clean_blank($1); + } elsif ($parametro =~ m/^ha_file\s(.*)/i) { $pa_config->{'ha_file'} = clean_blank($1); } + elsif ($parametro =~ m/^ha_hosts_file\s(.*)/i) { + $pa_config->{'ha_hosts_file'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_dbuser\s(.*)/i) { + $pa_config->{'ha_dbuser'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_dbpass\s(.*)/i) { + $pa_config->{'ha_dbpass'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_sshuser\s(.*)/i) { + $pa_config->{'ha_sshuser'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_sshport\s(.*)/i) { + $pa_config->{'ha_sshport'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_hosts\s(.*)/i) { + $pa_config->{'ha_hosts'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_resync\s(.*)/i) { + $pa_config->{'ha_resync'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_resync_log\s(.*)/i) { + $pa_config->{'ha_resync_log'} = clean_blank($1); + } elsif ($parametro =~ m/^ha_pid_file\s(.*)/i) { $pa_config->{'ha_pid_file'} = clean_blank($1); } @@ -1313,9 +1359,42 @@ sub pandora_load_config { elsif ($parametro =~ m/^dataserver_smart_queue\s([0-1])/i) { $pa_config->{'dataserver_smart_queue'} = clean_blank($1); } - + elsif ($parametro =~ m/^ha_connect_retries\s+([0-9]*)/i) { + $pa_config->{'ha_connect_retries'} = clean_blank($1); + } + elsif ($parametro =~ m/^ha_connect_delay\s+([0-9]*)/i) { + $pa_config->{'ha_connect_delay'} = clean_blank($1); + } + elsif ($parametro =~ m/^repl_dbuser\s(.*)/i) { + $pa_config->{'repl_dbuser'} = clean_blank($1); + } + elsif ($parametro =~ m/^repl_dbpass\s(.*)/i) { + $pa_config->{'repl_dbpass'} = clean_blank($1); + } } # end of loop for parameter # + # The DB host was overridden by pandora_ha. + if (-f $pa_config->{'ha_hosts_file'}) { + eval { + open(my $fh, '<', $pa_config->{'ha_hosts_file'}) or return; + my $dbhost = <$fh>; + chomp($dbhost); + if (defined($dbhost) && $dbhost ne '') { + $pa_config->{'dbhost'} = $dbhost; + } + close($fh); + }; + } + print " [*] DB Host is " . $pa_config->{'dbhost'} . "\n"; + + # ha_dbuser and ha_dbpass default to dbuser and dbpass respectively. + $pa_config->{'ha_dbuser'} = $pa_config->{'dbuser'} unless defined($pa_config->{'ha_dbuser'}); + $pa_config->{'ha_dbpass'} = $pa_config->{'dbpass'} unless defined($pa_config->{'ha_dbpass'}); + + # repl_dbuser and repl_dbpass default to dbuser and dbpass respectively. + $pa_config->{'repl_dbuser'} = $pa_config->{'dbuser'} unless defined($pa_config->{'repl_dbuser'}); + $pa_config->{'repl_dbpass'} = $pa_config->{'dbpass'} unless defined($pa_config->{'repl_dbpass'}); + # Generate the encryption key after reading the passphrase. $pa_config->{"encryption_key"} = enterprise_hook('pandora_get_encryption_key', [$pa_config, $pa_config->{"encryption_passphrase"}]); diff --git a/pandora_server/lib/PandoraFMS/Core.pm b/pandora_server/lib/PandoraFMS/Core.pm index b911f05b50..4726df4710 100644 --- a/pandora_server/lib/PandoraFMS/Core.pm +++ b/pandora_server/lib/PandoraFMS/Core.pm @@ -54,6 +54,8 @@ Exported Functions: =item * C +=item * C + =item * C =item * C @@ -194,6 +196,7 @@ our @EXPORT = qw( pandora_evaluate_alert pandora_evaluate_snmp_alerts pandora_event + pandora_timed_event pandora_extended_event pandora_execute_alert pandora_execute_action @@ -3996,7 +3999,8 @@ Generate an event. =cut ########################################################################## -sub pandora_event ($$$$$$$$$$;$$$$$$$$$$$$) { +#sub pandora_event ($$$$$$$$$$;$$$$$$$$$$$$) { +sub pandora_event { my ($pa_config, $evento, $id_grupo, $id_agente, $severity, $id_alert_am, $id_agentmodule, $event_type, $event_status, $dbh, $source, $user_name, $comment, $id_extra, $tags, @@ -4113,6 +4117,28 @@ sub pandora_event ($$$$$$$$$$;$$$$$$$$$$$$) { return $event_id; } +########################################################################## +=head2 C<< pandora_timed_event (I<$time_limit>, I<@event>) >> + +Generate an event, but no more than one every $time_limit seconds. + +=cut +########################################################################## +my %TIMED_EVENTS :shared; +sub pandora_timed_event ($@) { + my ($time_limit, @event) = @_; + + # Match events by message. + my $event_msg = $event[1]; + + # Do not generate more than one event every $time_limit seconds. + my $now = time(); + if (!defined($TIMED_EVENTS{$event_msg}) || $TIMED_EVENTS{$event_msg} + $time_limit < $now) { + $TIMED_EVENTS{$event_msg} = $now; + pandora_event(@event); + } +} + ########################################################################## =head2 C<< pandora_extended_event (I<$pa_config>, I<$dbh>, I<$event_id>, I<$description>) >> diff --git a/pandora_server/util/pandora_db.pl b/pandora_server/util/pandora_db.pl index 60699c5dbe..410f87c997 100755 --- a/pandora_server/util/pandora_db.pl +++ b/pandora_server/util/pandora_db.pl @@ -629,6 +629,21 @@ sub pandora_load_config_pdb ($) { $conf->{'errorlogfile'} = $conf->{'errorlog_file'}; $conf->{'errorlogfile'} = "/var/log/pandora_server.error" unless defined ($conf->{'errorlogfile'}); + # The DB host was overridden by pandora_ha. + $conf->{'ha_hosts_file'} = '/var/spool/pandora/data_in/conf/pandora_ha_hosts.conf' unless defined($conf->{'ha_hosts_file'}); + if (-f $conf->{'ha_hosts_file'}) { + eval { + open(my $fh, '<', $conf->{'ha_hosts_file'}) or return; + my $dbhost = <$fh>; + chomp($dbhost); + if (defined($dbhost) && $dbhost ne '') { + $conf->{'dbhost'} = $dbhost; + } + close($fh); + }; + } + print " [*] DB Host is " . $conf->{'dbhost'} . "\n"; + # Read additional tokens from the DB my $dbh = db_connect ($conf->{'dbengine'}, $conf->{'dbname'}, $conf->{'dbhost'}, $conf->{'dbport'}, $conf->{'dbuser'}, $conf->{'dbpass'}); diff --git a/pandora_server/util/pandora_ha.pl b/pandora_server/util/pandora_ha.pl index 9e44812908..cac883da54 100755 --- a/pandora_server/util/pandora_ha.pl +++ b/pandora_server/util/pandora_ha.pl @@ -34,15 +34,27 @@ my %Opts; # Run as a daemon. my $DAEMON = 0; +# Timeout for the HA DB lock. +my $LOCK_TIMEOUT = 300; + # Avoid retry old processing orders. my $First_Cleanup = 1; +# List of known HA DB hosts. +my @HA_DB_Hosts; + +# Current master node. +my $DB_Host = ''; + # PID file. my $PID_FILE = '/var/run/pandora_ha.pid'; # Server service handler. my $Pandora_Service; +# Restart the Pandora FMS Server. +my $Restart = 0; + # Controlled exit my $Running = 0; @@ -55,6 +67,10 @@ sub log_message($$$;$) { my $level = $verbosity_level; $level = 5 unless defined($level); + if ($source eq 'DEBUG' && !defined($ENV{'PANDORA_DEBUG'})) { + return; + } + if (ref($conf) eq "HASH") { logger($conf, 'HA (' . $source . ') ' . "$message", $level); } @@ -169,6 +185,17 @@ sub ha_keep_pandora_running($$) { my ($conf, $dbh) = @_; my $OSNAME = $^O; my $control_command; + $Pandora_Service = $conf->{'pandora_service_cmd'}; + + # A restart was requested. + if ($Restart == 1) { + $Restart = 0; + + log_message($conf, 'LOG', 'Restarting Pandora service'); + $control_command = $^O eq "freebsd" ? "restart_server" : 'restart-server'; + `$Pandora_Service $control_command $ENV{'PANDORA_DBHOST'} 2>/dev/null`; + return; + } # Check if all servers are running # Restart if crashed or keep interval is over. @@ -185,8 +212,6 @@ sub ha_keep_pandora_running($$) { my $nservers = get_db_value ($dbh, 'SELECT count(*) FROM tserver where name = ?', $conf->{'servername'}); - $Pandora_Service = $conf->{'pandora_service_cmd'}; - # Check if service is running $control_command = "status-server"; if ($OSNAME eq "freebsd") { @@ -308,6 +333,37 @@ sub ha_update_server($$) { } +################################################################################ +# Dump the list of known databases to disk. +################################################################################ +sub ha_dump_databases($) { + my ($conf) = @_; + + # HA is not configured. + return unless defined($conf->{'ha_hosts'}); + + eval { + open(my $fh, '>', $conf->{'ha_hosts_file'}); + print $fh $DB_Host; # The console only needs the master DB. + close($fh); + log_message($conf, 'DEBUG', "Dumped master database $DB_Host to disk"); + }; + log_message($conf, 'WARNING', $@) if ($@); +} + +################################################################################ +# Read the list of known databases from disk. +################################################################################ +sub ha_load_databases($) { + my ($conf) = @_; + + # HA is not configured. + return unless defined($conf->{'ha_hosts'}); + + @HA_DB_Hosts = grep { !/^#/ } map { s/^\s+|\s+$//g; $_; } split(/,/, $conf->{'ha_hosts'}); + log_message($conf, 'DEBUG', "Loaded databases from disk (@HA_DB_Hosts)"); +} + ############################################################################### # Connect to ha database, falling back to direct connection to db. ############################################################################### @@ -324,9 +380,111 @@ sub ha_database_connect($) { } ############################################################################### -# Main +# Connect to ha database, falling back to direct connection to db. ############################################################################### -sub ha_main($) { +sub ha_database_connect_pandora($) { + my $conf = shift; + my $dbhost = $conf->{'dbhost'}; + + # Load the list of HA databases. + ha_load_databases($conf); + + # Select a new master database. + my ($dbh, $utimestamp, $max_utimestamp) = (undef, undef, -1); + foreach my $ha_dbhost (@HA_DB_Hosts) { + + # Retry each database ha_connect_retries times. + for (my $i = 0; $i < $conf->{'ha_connect_retries'}; $i++) { + eval { + log_message($conf, 'DEBUG', "Trying database $ha_dbhost..."); + $dbh= db_connect('mysql', + $conf->{'dbname'}, + $ha_dbhost, + $conf->{'dbport'}, + $conf->{'ha_dbuser'}, + $conf->{'ha_dbpass'}); + log_message($conf, 'DEBUG', "Connected to database $ha_dbhost"); + }; + log_message($conf, 'WARNING', $@) if ($@); + + # Connection successful. + last if defined($dbh); + + # Wait for the next retry. + sleep($conf->{'ha_connect_delay'}); + } + + # No luck. Try the next database. + next unless defined($dbh); + + eval { + # Get the most recent utimestamp from the database. + $utimestamp = get_db_value($dbh, 'SELECT UNIX_TIMESTAMP(MAX(keepalive)) FROM tserver'); + db_disconnect($dbh); + + # Did we find a more recent database? + $utimestamp = 0 unless defined($utimestamp); + if ($utimestamp > $max_utimestamp) { + $dbhost = $ha_dbhost; + $max_utimestamp = $utimestamp; + } + }; + log_message($conf, 'WARNING', $@) if ($@); + } + + # Return a connection to the selected master. + eval { + log_message($conf, 'DEBUG', "Connecting to selected master $dbhost..."); + $dbh = db_connect('mysql', + $conf->{'dbname'}, + $dbhost, + $conf->{'dbport'}, + $conf->{'ha_dbuser'}, + $conf->{'ha_dbpass'}); + + # Restart if a new master was selected. + if ($dbhost ne $DB_Host) { + log_message($conf, 'DEBUG', "Setting master database to $dbhost"); + $DB_Host = $dbhost; + $Restart = 1; + } + }; + log_message($conf, 'WARNING', $@) if ($@); + + # Save the list of HA databases. + ha_dump_databases($conf); + + return $dbh; +} + +############################################################################### +# Return 1 if the given DB is read-only, 0 otherwise. +############################################################################### +sub ha_read_only($$) { + my ($conf, $dbh) = @_; + + my $read_only = get_db_value($dbh, 'SELECT @@global.read_only'); + return 1 if (defined($read_only) && $read_only == 1); + + return 0; +} + +############################################################################### +# Restart the Pandora FMS Server. +############################################################################### +sub ha_restart_pandora($) { + my ($config) = @_; + + my $control_command = $^O eq 'freebsd' ? + 'restart_server' : + 'restart-server'; + `$config->{'pandora_service_cmd'} $control_command 2>/dev/null`; +} + +############################################################################### +# Main (Pacemaker) +############################################################################### +sub ha_main_pacemaker($) { my ($conf) = @_; # Set the PID file. @@ -404,6 +562,100 @@ sub ha_main($) { } } +############################################################################### +# Main (Pandora) +############################################################################### +sub ha_main_pandora($) { + my ($conf) = @_; + + # Set the PID file. + $conf->{'PID'} = $PID_FILE; + + # Log to a separate file if needed. + $conf->{'log_file'} = $conf->{'ha_log_file'} if defined ($conf->{'ha_log_file'}); + + # Run in the background. + ha_daemonize($conf) if ($DAEMON == 1); + + # Main loop. + $Running = 1; + while ($Running) { + my $dbh = undef; + eval { + + # Connect to a DB. + log_message($conf, 'LOG', "Looking for databases"); + $dbh = ha_database_connect_pandora($conf); + if (!defined($dbh)) { + log_message($conf, 'LOG', 'No databases available'); + return; + } + + # Make the DB host available to the Pandora FMS Server. + $ENV{'PANDORA_DBHOST'} = $DB_Host; + + # Needed for the Enterprise module. + $conf->{'dbhost'} = $DB_Host; + + # Enterprise capabilities need access to the DB. + eval { + local $SIG{__DIE__}; + # Load enterprise components. + enterprise_load($conf, 1); + + # Register Enterprise logger + enterprise_hook('pandoraha_logger', [\&log_message]); + log_message($conf, 'LOG', 'Enterprise capabilities loaded'); + + }; + log_message($conf, 'LOG', "No enterprise capabilities: $@") if ($@); + + log_message($conf, 'LOG', "Connected to database $DB_Host"); + enterprise_hook('pandoraha_stop_slave', [$conf, $dbh]); + + if (ha_read_only($conf, $dbh) == 1) { + log_message($conf, 'LOG', "The database is read-only."); + return; + } + + # Check if there are updates pending. + ha_update_server($conf, $dbh); + + # Keep pandora running + ha_keep_pandora_running($conf, $dbh); + + # Keep Tentacle running + ha_keep_tentacle_running($conf, $dbh); + + # Are we the master? + pandora_set_master($conf, $dbh); + if (!pandora_is_master($conf)) { + log_message($conf, 'LOG', $conf->{'servername'} . ' is not the current master. Skipping DB-HA actions and monitoring.'); + return; + } + + # Check the status of slave databases. + enterprise_hook('pandoraha_check_slaves', [$conf, $dbh, $DB_Host, \@HA_DB_Hosts]); + + # Update the status of HA databases. + enterprise_hook('pandoraha_update_dbs', [$conf, $dbh, $DB_Host, \@HA_DB_Hosts]); + + # Execute resync actions. + enterprise_hook('pandoraha_resync_dbs', [$conf, $dbh, $DB_Host, \@HA_DB_Hosts]); + }; + log_message($conf, 'WARNING', $@) if ($@); + + # Cleanup. + eval { + db_disconnect($dbh) if defined($dbh); + }; + + # Go to sleep. + log_message($conf, 'LOG', "Sleep."); + sleep($conf->{'ha_interval'}); + } +} + ################################################################################ # Stop pandora server ################################################################################ @@ -429,7 +681,6 @@ END { stop(); } - $SIG{INT} = \&stop; $SIG{TERM} = \&stop; @@ -440,6 +691,10 @@ ha_init_pandora(\%Conf); ha_load_pandora_conf (\%Conf); # Main -ha_main(\%Conf); +if (defined($Conf{'ha_mode'}) && lc($Conf{'ha_mode'}) eq 'pandora') { + ha_main_pandora(\%Conf); +} else { + ha_main_pacemaker(\%Conf); +} exit 0;