From 1f3b95f6d155b03088889451b46cee8c3b5ad590 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 8 Mar 2018 12:09:03 +0100 Subject: [PATCH 1/7] check problems migration --- pandora_console/include/functions_api.php | 21 +++++++++++++++++++++ pandora_console/pandoradb.sql | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/pandora_console/include/functions_api.php b/pandora_console/include/functions_api.php index e6d247b421..23c0a4d239 100644 --- a/pandora_console/include/functions_api.php +++ b/pandora_console/include/functions_api.php @@ -34,6 +34,7 @@ enterprise_include_once ('include/functions_local_components.php'); enterprise_include_once ('include/functions_events.php'); enterprise_include_once ('include/functions_agents.php'); enterprise_include_once ('include/functions_modules.php'); +enterprise_include_once ('include/functions_collection.php'); /** * Parse the "other" parameter. @@ -10142,4 +10143,24 @@ function api_set_metaconsole_synch($keys) { } +/** + * Returns the collections associated to this agent + */ +function api_get_agent_collections($id_agent) { + // ACL! + $collections = enterprise_hook("collection_get_collections_in_agent", array($id_agent)); + echo json_encode($collections); +} + +/** + * Returns if collection exists in this node + */ +function api_get_check_collection_id($id_collection) { + // ACL! + $exists = enterprise_hook("collection_exits_directory", array($id_collection)); + echo json_encode($exists); +} + + + ?> diff --git a/pandora_console/pandoradb.sql b/pandora_console/pandoradb.sql index b43432c7b4..aaf1ef6eb8 100644 --- a/pandora_console/pandoradb.sql +++ b/pandora_console/pandoradb.sql @@ -3081,3 +3081,26 @@ create table IF NOT EXISTS `tcluster_agent`( ON UPDATE CASCADE ) engine=InnoDB DEFAULT CHARSET=utf8; +-- --------------------------------------------------------------------- +-- Table `tmigration_queue` +-- --------------------------------------------------------------------- + +create table IF NOT EXISTS `tmigration_queue`( + `id_metaconsole_agent` int unsigned not null, + `id_source_node` int unsigned not null, + `id_target_node` int unsigned not null, + `priority` int unsigned not null default 0, + `step` int unsigned not null default 0 +) engine=InnoDB DEFAULT CHARSET=utf8; + +-- --------------------------------------------------------------------- +-- Table `tmigration_module_queue` +-- --------------------------------------------------------------------- + +create table IF NOT EXISTS `tmigration_module_queue`( + `id_source_node` int unsigned not null, + `id_target_node` int unsigned not null, + `id_source_agentmodule` int unsigned not null, + `id_target_agentmodule` int unsigned not null, + `last_replication_timestamp` bigint(20) NOT NULL default 0 +) engine=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file From 2605064c025b4e9d5910573d76c24f63e76652fa Mon Sep 17 00:00:00 2001 From: daniel Date: Fri, 9 Mar 2018 15:33:51 +0100 Subject: [PATCH 2/7] minor change --- pandora_console/pandoradb.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandora_console/pandoradb.sql b/pandora_console/pandoradb.sql index aaf1ef6eb8..7528d99f04 100644 --- a/pandora_console/pandoradb.sql +++ b/pandora_console/pandoradb.sql @@ -3086,11 +3086,11 @@ create table IF NOT EXISTS `tcluster_agent`( -- --------------------------------------------------------------------- create table IF NOT EXISTS `tmigration_queue`( - `id_metaconsole_agent` int unsigned not null, + `id_metaconsole_agent` int unsigned not null auto_increment, `id_source_node` int unsigned not null, `id_target_node` int unsigned not null, - `priority` int unsigned not null default 0, - `step` int unsigned not null default 0 + `priority` int unsigned default 0, + `step` int unsigned default 0 ) engine=InnoDB DEFAULT CHARSET=utf8; -- --------------------------------------------------------------------- From c8118c2e08ddec97c57bb5a1f8fe2a4aacc201c4 Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 12 Mar 2018 19:31:24 +0100 Subject: [PATCH 3/7] fixed minor error --- pandora_console/include/functions_api.php | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pandora_console/include/functions_api.php b/pandora_console/include/functions_api.php index 23c0a4d239..c3fbfef69b 100644 --- a/pandora_console/include/functions_api.php +++ b/pandora_console/include/functions_api.php @@ -10143,24 +10143,4 @@ function api_set_metaconsole_synch($keys) { } -/** - * Returns the collections associated to this agent - */ -function api_get_agent_collections($id_agent) { - // ACL! - $collections = enterprise_hook("collection_get_collections_in_agent", array($id_agent)); - echo json_encode($collections); -} - -/** - * Returns if collection exists in this node - */ -function api_get_check_collection_id($id_collection) { - // ACL! - $exists = enterprise_hook("collection_exits_directory", array($id_collection)); - echo json_encode($exists); -} - - - ?> From 7d58734ef26abf825e4bdf16aa5ea40d6f7ebfc8 Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Mon, 12 Mar 2018 19:39:28 +0100 Subject: [PATCH 4/7] MigrationServer aux util --- pandora_console/pandoradb.sql | 27 +++++++++++++------ pandora_server/lib/PandoraFMS/DB.pm | 24 +++++++++++++++++ pandora_server/lib/PandoraFMS/Tools.pm | 37 ++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/pandora_console/pandoradb.sql b/pandora_console/pandoradb.sql index 7528d99f04..bafb2e309f 100644 --- a/pandora_console/pandoradb.sql +++ b/pandora_console/pandoradb.sql @@ -3086,11 +3086,15 @@ create table IF NOT EXISTS `tcluster_agent`( -- --------------------------------------------------------------------- create table IF NOT EXISTS `tmigration_queue`( - `id_metaconsole_agent` int unsigned not null auto_increment, + `id` int unsigned not null auto_increment, + `id_source_agent` int unsigned not null, + `id_target_agent` int unsigned not null, `id_source_node` int unsigned not null, - `id_target_node` int unsigned not null, - `priority` int unsigned default 0, - `step` int unsigned default 0 + `id_target_node` int unsigned not null, + `priority` int unsigned default 0, + `step` int unsigned default 0, + `running` tinyint(2) unsigned not null default 0, + PRIMARY KEY(`id`) ) engine=InnoDB DEFAULT CHARSET=utf8; -- --------------------------------------------------------------------- @@ -3098,9 +3102,16 @@ create table IF NOT EXISTS `tmigration_queue`( -- --------------------------------------------------------------------- create table IF NOT EXISTS `tmigration_module_queue`( + `id` int unsigned not null auto_increment, + `id_migration` int unsigned not null, `id_source_node` int unsigned not null, `id_target_node` int unsigned not null, - `id_source_agentmodule` int unsigned not null, - `id_target_agentmodule` int unsigned not null, - `last_replication_timestamp` bigint(20) NOT NULL default 0 -) engine=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file + `id_source_agentmodule` int unsigned not null, + `id_target_agentmodule` int unsigned not null, + `last_replication_timestamp` bigint(20) NOT NULL default 0, + PRIMARY KEY(`id`), + FOREIGN KEY(`id_migration`) REFERENCES tmigration_queue(`id`) + ON DELETE CASCADE + ON UPDATE CASCADE +) engine=InnoDB DEFAULT CHARSET=utf8; + diff --git a/pandora_server/lib/PandoraFMS/DB.pm b/pandora_server/lib/PandoraFMS/DB.pm index 4176a47d94..ee032e15e1 100644 --- a/pandora_server/lib/PandoraFMS/DB.pm +++ b/pandora_server/lib/PandoraFMS/DB.pm @@ -40,6 +40,7 @@ our @EXPORT = qw( db_get_lock db_insert db_insert_get_values + db_insert_from_hash db_process_insert db_process_update db_release_lock @@ -900,6 +901,29 @@ sub db_process_insert($$$$;@) { return $res; } +######################################################################## +## SQL insert from hash +## 1st: dbh +## 2nd: table name, +## 2nd: {field => value} ref +######################################################################## +sub db_insert_from_hash { + my ($dbh, $index, $table, $data) = @_; + + my @fields = keys %{$data}; + my $values_prep = ""; + my @values = values %{$data}; + my $nfields = scalar @fields; + + for (my $i=0; $i<$nfields; $i++) { + $values_prep .= "?,"; + } + $values_prep =~ s/,$//; + + return db_insert($dbh, $index, "INSERT INTO " . $table . " (" . join (",", @fields) . ") VALUES ($values_prep)", @values); + +} + ######################################################################## ## SQL update. ######################################################################## diff --git a/pandora_server/lib/PandoraFMS/Tools.pm b/pandora_server/lib/PandoraFMS/Tools.pm index fe528042b8..ff6763c13a 100755 --- a/pandora_server/lib/PandoraFMS/Tools.pm +++ b/pandora_server/lib/PandoraFMS/Tools.pm @@ -27,6 +27,7 @@ use Encode; use Socket qw(inet_ntoa inet_aton); use Sys::Syslog; use Scalar::Util qw(looks_like_number); +use LWP::UserAgent; # New in 3.2. Used to sendmail internally, without external scripts # use Module::Loaded; @@ -62,8 +63,9 @@ our @EXPORT = qw( TRANSACTIONALSERVER SYNCSERVER SYSLOGSERVER - METACONSOLE_LICENSE WUXSERVER + MIGRATIONSERVER + METACONSOLE_LICENSE $DEVNULL $OS $OS_VERSION @@ -74,7 +76,8 @@ our @EXPORT = qw( MODULE_WARNING MODULE_UNKNOWN MODULE_NOTINIT - cron_get_closest_in_range + api_call_url + cron_get_closest_in_range cron_next_execution cron_next_execution_date cron_check_syntax @@ -133,6 +136,7 @@ use constant MFSERVER => 15; use constant SYNCSERVER => 16; use constant WUXSERVER => 17; use constant SYSLOGSERVER => 18; +use constant MIGRATIONSERVER => 19; # Module status use constant MODULE_NORMAL => 0; @@ -1708,6 +1712,35 @@ sub uri_encode_literal_percent { } ## end sub uri_encode_literal_percent +################################################################################ +# Launch API call +################################################################################ +sub api_call_url { + my ($pa_config, $server_url, $api_params, @options) = @_; + + + my $ua = LWP::UserAgent->new(); + $ua->timeout($options->{lwp_timeout}); + # Enable environmental proxy settings + $ua->env_proxy; + # Enable in-memory cookie management + $ua->cookie_jar( {} ); + + # Disable verify host certificate (only needed for self-signed cert) + $ua->ssl_opts( 'verify_hostname' => 0 ); + $ua->ssl_opts( 'SSL_verify_mode' => 0x00 ); + + my $response; + + eval { + $response = $ua->post($server_url, $api_params, @options); + }; + if ((!$@) && $response->is_success) { + return $response->decoded_content; + } + return undef; +} + # End of function declaration # End of defined Code From ed282c6a9b6450df9b753856c782120cedc0cf5e Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 13 Mar 2018 16:35:13 +0100 Subject: [PATCH 5/7] 1923 Added required methods for migrationserver --- pandora_console/pandoradb.sql | 7 ++-- pandora_server/bin/pandora_server | 2 + pandora_server/lib/PandoraFMS/DB.pm | 61 +++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/pandora_console/pandoradb.sql b/pandora_console/pandoradb.sql index bafb2e309f..a02660b431 100644 --- a/pandora_console/pandoradb.sql +++ b/pandora_console/pandoradb.sql @@ -3092,8 +3092,9 @@ create table IF NOT EXISTS `tmigration_queue`( `id_source_node` int unsigned not null, `id_target_node` int unsigned not null, `priority` int unsigned default 0, - `step` int unsigned default 0, - `running` tinyint(2) unsigned not null default 0, + `step` int default 0, + `running` tinyint(2) default 0, + `active_db_only` tinyint(2) default 0, PRIMARY KEY(`id`) ) engine=InnoDB DEFAULT CHARSET=utf8; @@ -3104,8 +3105,6 @@ create table IF NOT EXISTS `tmigration_queue`( create table IF NOT EXISTS `tmigration_module_queue`( `id` int unsigned not null auto_increment, `id_migration` int unsigned not null, - `id_source_node` int unsigned not null, - `id_target_node` int unsigned not null, `id_source_agentmodule` int unsigned not null, `id_target_agentmodule` int unsigned not null, `last_replication_timestamp` bigint(20) NOT NULL default 0, diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index 1f13627cb3..900565095a 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -37,6 +37,7 @@ use PandoraFMS::ReconServer; use PandoraFMS::WMIServer; use PandoraFMS::PluginServer; use PandoraFMS::PredictionServer; +use PandoraFMS::MigrationServer; # Constants for Win32 services. use constant WIN32_SERVICE_STOPPED => 0x01; @@ -118,6 +119,7 @@ sub pandora_startup () { } else { # Metaconsole service modules are run by the prediction server push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH)); + push (@Servers, new PandoraFMS::MigrationServer (\%Config, $DBH)); } # There are enterprise metaconsole servers! diff --git a/pandora_server/lib/PandoraFMS/DB.pm b/pandora_server/lib/PandoraFMS/DB.pm index ee032e15e1..1fe1208d78 100644 --- a/pandora_server/lib/PandoraFMS/DB.pm +++ b/pandora_server/lib/PandoraFMS/DB.pm @@ -34,12 +34,14 @@ our @EXPORT = qw( add_new_address_agent db_concat db_connect + db_history_connect db_delete_limit db_disconnect db_do db_get_lock db_insert db_insert_get_values + db_insert_from_array_hash db_insert_from_hash db_process_insert db_process_update @@ -168,6 +170,28 @@ sub db_connect ($$$$$$) { return undef; } +########################################################################## +## Connect to a history DB associated to given dbh. +########################################################################## +sub db_history_connect { + my ($dbh, $pa_config) = @_; + + my %conf; + + $conf{'history_db_enabled'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_enabled"); + $conf{'history_db_host'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_host"); + $conf{'history_db_port'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_port"); + $conf{'history_db_name'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_name"); + $conf{'history_db_user'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_user"); + $conf{'history_db_pass'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = ?", "history_db_pass"); + + my $history_dbh = ($conf{'history_db_enabled'} eq '1') ? db_connect ($pa_config->{'dbengine'}, $conf{'history_db_name'}, + $conf{'history_db_host'}, $conf{'history_db_port'}, $conf{'history_db_user'}, $conf{'history_db_pass'}) : undef; + + + return $history_dbh; +} + ######################################################################## ## Disconnect from the DB. ######################################################################## @@ -904,14 +928,15 @@ sub db_process_insert($$$$;@) { ######################################################################## ## SQL insert from hash ## 1st: dbh -## 2nd: table name, -## 2nd: {field => value} ref +## 2nd: index +## 3rd: table name, +## 4th: {field => value} ref ######################################################################## sub db_insert_from_hash { my ($dbh, $index, $table, $data) = @_; - my @fields = keys %{$data}; my $values_prep = ""; + my @fields = keys %{$data}; my @values = values %{$data}; my $nfields = scalar @fields; @@ -921,7 +946,37 @@ sub db_insert_from_hash { $values_prep =~ s/,$//; return db_insert($dbh, $index, "INSERT INTO " . $table . " (" . join (",", @fields) . ") VALUES ($values_prep)", @values); +} +######################################################################## +## SQL insert from hash +## 1st: dbh +## 2nd: index +## 3rd: table name, +## 4th: array({field => value},{field => value}) array ref +## +## Returns: An array with the inserted indexes +######################################################################## +sub db_insert_from_array_hash { + my ($dbh, $index, $table, $data) = @_; + + if ((!defined($data) || ref ($data) ne "ARRAY")) { + return (); + } + + + my @inserted_keys; + + eval { + foreach my $row (@{$data}) { + push @inserted_keys, db_insert_from_hash($dbh, $index, $table, $row); + } + }; + if ($@) { + return undef; + } + + return @inserted_keys; } ######################################################################## From 9b9b46ac65b4b7b19f09c544c5716f00f4c1b71b Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 13 Mar 2018 16:38:09 +0100 Subject: [PATCH 6/7] 1923 added new tables to MR and migrate sql script --- pandora_console/extras/mr/13.sql | 25 +++++++++++++ .../pandoradb_migrate_6.0_to_7.0.mysql.sql | 35 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/pandora_console/extras/mr/13.sql b/pandora_console/extras/mr/13.sql index 96aae03cb9..ccf8b35725 100644 --- a/pandora_console/extras/mr/13.sql +++ b/pandora_console/extras/mr/13.sql @@ -12,4 +12,29 @@ START TRANSACTION; INSERT INTO `tconfig_os` (`id_os`, `name`, `description`, `icon_name`) VALUES (21, 'Cluster', 'Cluster agent', 'so_cluster.png'); + create table IF NOT EXISTS `tmigration_queue`( + `id` int unsigned not null auto_increment, + `id_source_agent` int unsigned not null, + `id_target_agent` int unsigned not null, + `id_source_node` int unsigned not null, + `id_target_node` int unsigned not null, + `priority` int unsigned default 0, + `step` int default 0, + `running` tinyint(2) default 0, + `active_db_only` tinyint(2) default 0, + PRIMARY KEY(`id`) + ) engine=InnoDB DEFAULT CHARSET=utf8; + + create table IF NOT EXISTS `tmigration_module_queue`( + `id` int unsigned not null auto_increment, + `id_migration` int unsigned not null, + `id_source_agentmodule` int unsigned not null, + `id_target_agentmodule` int unsigned not null, + `last_replication_timestamp` bigint(20) NOT NULL default 0, + PRIMARY KEY(`id`), + FOREIGN KEY(`id_migration`) REFERENCES tmigration_queue(`id`) + ON DELETE CASCADE + ON UPDATE CASCADE + ) engine=InnoDB DEFAULT CHARSET=utf8; + COMMIT; \ No newline at end of file diff --git a/pandora_console/extras/pandoradb_migrate_6.0_to_7.0.mysql.sql b/pandora_console/extras/pandoradb_migrate_6.0_to_7.0.mysql.sql index 4e292b4f0d..67c6847423 100644 --- a/pandora_console/extras/pandoradb_migrate_6.0_to_7.0.mysql.sql +++ b/pandora_console/extras/pandoradb_migrate_6.0_to_7.0.mysql.sql @@ -1533,3 +1533,38 @@ create table IF NOT EXISTS `tcluster_agent`( FOREIGN KEY (`id_cluster`) REFERENCES tcluster(`id`) ON UPDATE CASCADE ) engine=InnoDB DEFAULT CHARSET=utf8; + + +-- --------------------------------------------------------------------- +-- Table `tmigration_queue` +-- --------------------------------------------------------------------- + +create table IF NOT EXISTS `tmigration_queue`( + `id` int unsigned not null auto_increment, + `id_source_agent` int unsigned not null, + `id_target_agent` int unsigned not null, + `id_source_node` int unsigned not null, + `id_target_node` int unsigned not null, + `priority` int unsigned default 0, + `step` int default 0, + `running` tinyint(2) default 0, + `active_db_only` tinyint(2) default 0, + PRIMARY KEY(`id`) +) engine=InnoDB DEFAULT CHARSET=utf8; + +-- --------------------------------------------------------------------- +-- Table `tmigration_module_queue` +-- --------------------------------------------------------------------- + +create table IF NOT EXISTS `tmigration_module_queue`( + `id` int unsigned not null auto_increment, + `id_migration` int unsigned not null, + `id_source_agentmodule` int unsigned not null, + `id_target_agentmodule` int unsigned not null, + `last_replication_timestamp` bigint(20) NOT NULL default 0, + PRIMARY KEY(`id`), + FOREIGN KEY(`id_migration`) REFERENCES tmigration_queue(`id`) + ON DELETE CASCADE + ON UPDATE CASCADE +) engine=InnoDB DEFAULT CHARSET=utf8; + From a6c5a0b046f4fafa983ab72278344e6483d1eb52 Mon Sep 17 00:00:00 2001 From: fbsanchez Date: Tue, 13 Mar 2018 17:26:44 +0100 Subject: [PATCH 7/7] Moved MigrationServer to enterprise servers load --- pandora_server/bin/pandora_server | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandora_server/bin/pandora_server b/pandora_server/bin/pandora_server index 900565095a..1f13627cb3 100755 --- a/pandora_server/bin/pandora_server +++ b/pandora_server/bin/pandora_server @@ -37,7 +37,6 @@ use PandoraFMS::ReconServer; use PandoraFMS::WMIServer; use PandoraFMS::PluginServer; use PandoraFMS::PredictionServer; -use PandoraFMS::MigrationServer; # Constants for Win32 services. use constant WIN32_SERVICE_STOPPED => 0x01; @@ -119,7 +118,6 @@ sub pandora_startup () { } else { # Metaconsole service modules are run by the prediction server push (@Servers, new PandoraFMS::PredictionServer (\%Config, $DBH)); - push (@Servers, new PandoraFMS::MigrationServer (\%Config, $DBH)); } # There are enterprise metaconsole servers!