Merge remote-tracking branch 'origin/ent-4226-mejoras-plugin-sap' into ent-EDF

Conflicts:
	pandora_console/extras/mr/33.sql
	pandora_console/godmode/wizards/Wizard.main.php
This commit is contained in:
fbsanchez 2019-11-14 21:40:35 +01:00
commit f69b86663a
15 changed files with 417 additions and 69 deletions

View File

@ -36,4 +36,6 @@ CREATE TABLE `tremote_command_target` (
INSERT INTO `tconfig`(`token`, `value`) VALUES ('welcome_state', -1);
ALTER TABLE `tcredential_store` MODIFY COLUMN `product` enum('CUSTOM', 'AWS', 'AZURE', 'GOOGLE', 'SAP') default 'CUSTOM';
COMMIT;

View File

@ -2246,7 +2246,7 @@ CREATE TABLE `tvisual_console_elements_cache` (
CREATE TABLE IF NOT EXISTS `tcredential_store` (
`identifier` varchar(100) NOT NULL,
`id_group` mediumint(4) unsigned NOT NULL DEFAULT 0,
`product` enum('CUSTOM', 'AWS', 'AZURE', 'GOOGLE') default 'CUSTOM',
`product` enum('CUSTOM', 'AWS', 'AZURE', 'GOOGLE', 'SAP') default 'CUSTOM',
`username` text,
`password` text,
`extra_1` text,

View File

@ -0,0 +1,77 @@
<?php
/**
* Credential store
*
* @category HelperFeedBack
* @package Pandora FMS
* @subpackage Help Feedback
* @version 1.0.0
* @license See below
*
* ______ ___ _______ _______ ________
* | __ \.-----.--.--.--| |.-----.----.-----. | ___| | | __|
* | __/| _ | | _ || _ | _| _ | | ___| |__ |
* |___| |___._|__|__|_____||_____|__| |___._| |___| |__|_|__|_______|
*
* ============================================================================
* Copyright (c) 2005-2019 Artica Soluciones Tecnologicas
* Please see http://pandorafms.org for full contribution list
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation for version 2.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* ============================================================================
*/
// Begin.
/**
* Class sap_views.
*/
global $config;
enterprise_include_once('/include/class/SAPView.class.php');
$ajaxPage = 'general/sap_view';
// Control call flow.
try {
// User access and validation is being processed on class constructor.
$sap_views = new SAPView($ajaxPage);
} catch (Exception $e) {
if (is_ajax()) {
echo json_encode(['error' => '[sap_views]'.$e->getMessage() ]);
exit;
} else {
echo '[sap_views]'.$e->getMessage();
}
// Stop this execution, but continue 'globally'.
return;
}
// Ajax controller.
if (is_ajax()) {
$method = get_parameter('method', '');
if (method_exists($sap_views, $method) === true) {
if ($sap_views->ajaxMethod($method) === true) {
$sap_views->{$method}();
} else {
$sap_views->error('Unavailable method.');
}
} else {
$sap_views->error('Method not found. ['.$method.']');
}
// Stop any execution.
exit;
} else {
// Run.
$sap_views->run();
}

View File

@ -486,6 +486,20 @@ if ($id_agente) {
$agent_wizard['active'] = false;
}
$is_sap = agents_get_sap_agents($id_agente);
if ($is_sap) {
$saptab['text'] = '<a href="index.php?sec=gagente&sec2=godmode/agentes/configurar_agente&tab=sap_view&page=1&id_agente='.$id_agente.'">'.html_print_image('images/chart_curve.png', true, ['title' => __('SAP view')]).'</a>';
if ($tab == 'sap_view') {
$saptab['active'] = true;
} else {
$saptab['active'] = false;
}
} else {
$saptab = '';
}
$total_incidents = agents_get_count_incidents($id_agente);
// Incident tab.
@ -531,6 +545,7 @@ if ($id_agente) {
'group' => $grouptab,
'gis' => $gistab,
'agent_wizard' => $agent_wizard,
'sap_view' => $saptab,
];
} else {
$onheader = [
@ -546,6 +561,8 @@ if ($id_agente) {
'group' => $grouptab,
'gis' => $gistab,
'agent_wizard' => $agent_wizard,
'sap_view' => $saptab,
];
}
@ -691,6 +708,12 @@ if ($id_agente) {
}
break;
case 'sap_view':
$tab_description = '- '.__('SAP view');
$help_header = 'sap_view';
$tab_name = 'SAP View';
break;
default:
// Default.
break;
@ -2346,6 +2369,10 @@ switch ($tab) {
include 'agent_wizard.php';
break;
case 'sap_view':
include 'general/sap_view.php';
break;
default:
if (enterprise_hook('switch_agent_tab', [$tab])) {
// This will make sure that blank pages will have at least some

View File

@ -773,6 +773,9 @@ class DiscoveryTaskList extends Wizard
case DISCOVERY_CLOUD_AWS_RDS:
return 'wiz=cloud&mode=amazonws&ki='.$task['auth_strings'].'&sub=rds&page=0';
case DISCOVERY_APP_SAP:
return 'wiz=app&mode=SAP&page=0';
default:
if ($task['description'] == 'console_task') {
return 'wiz=ctask';

View File

@ -432,7 +432,7 @@ class CredentialStore extends Wizard
return $return;
}
return false;
return [];
}
@ -866,6 +866,7 @@ class CredentialStore extends Wizard
'CUSTOM' => __('Custom'),
'AWS' => __('Aws'),
'AZURE' => __('Azure'),
'SAP' => __('SAP'),
// 'GOOGLE' => __('Google'),
],
'selected' => (isset($values['product']) ? $values['product'] : 'CUSTOM'),
@ -900,6 +901,7 @@ class CredentialStore extends Wizard
case 'GOOGLE':
// Need further investigation.
case 'CUSTOM':
case 'SAP':
$user_label = __('Account ID');
$pass_label = __('Password');
$extra1 = false;
@ -1038,7 +1040,12 @@ class CredentialStore extends Wizard
$('#div-extra_2 label').text('<?php echo __('Subscription id'); ?>');
$('#div-extra_1').show();
$('#div-extra_2').show();
}
} else if ($('#product :selected').val() == "SAP") {
$('#div-username label').text('<?php echo __('Account ID.'); ?>');
$('#div-password label').text('<?php echo __('Password'); ?>');
$('#div-extra_1').hide();
$('#div-extra_2').hide();
}
}
/**

View File

@ -537,7 +537,7 @@ define('NODE_GENERIC', 3);
define('STATUS_OK', 0);
define('STATUS_ERROR', 1);
// Maps (new networkmaps and new visualmaps).
// Maps new networkmaps and new visualmaps.
define('MAP_TYPE_NETWORKMAP', 0);
define('MAP_TYPE_VISUALMAP', 1);
@ -590,6 +590,7 @@ define('DISCOVERY_CLOUD_AWS_EC2', 6);
define('DISCOVERY_CLOUD_AWS_RDS', 7);
define('DISCOVERY_CLOUD_AZURE_COMPUTE', 8);
define('DISCOVERY_DEPLOY_AGENTS', 9);
define('DISCOVERY_APP_SAP', 10);
// Discovery types matching definition.

View File

@ -3481,3 +3481,66 @@ function agents_get_status_animation($up=true)
);
}
}
/**
* Return if an agent is SAP or or an a agent SAP list.
* If function receive false, you will return all SAP agents,
* but if you receive an id agent, check if it is a sap agent
* and return true or false.
*
* @param integer $id_agent
* @return boolean
*/
function agents_get_sap_agents($id_agent)
{
// Available modules.
// If you add more modules, please update SAP.pm.
$sap_modules = [
160 => __('SAP Login OK'),
109 => __('SAP Dumps'),
111 => __('SAP List lock'),
113 => __('SAP Cancel Jobs'),
121 => __('SAP Batch input erroneus'),
104 => __('SAP Idoc erroneus'),
105 => __('SAP IDOC OK'),
150 => __('SAP WP without active restart'),
151 => __('SAP WP stopped'),
102 => __('Average time of SAPGUI response '),
180 => __('Dialog response time'),
103 => __('Dialog Logged users '),
192 => __('SYSFAIL, delivery attempts tRFC wrong entries number'),
195 => __('SYSFAIL, queue qRFC INPUT, wrong entries number '),
116 => __('Number of Update WPs in error'),
];
$array_agents = [];
foreach ($sap_modules as $module => $key) {
$array_agents = array_merge(
$array_agents,
db_get_all_rows_sql(
'SELECT ta.id_agente,ta.alias
FROM tagente ta
INNER JOIN tagente_modulo tam
ON tam.id_agente = ta.id_agente
WHERE tam.nombre
LIKE "%SAP%"
GROUP BY ta.id_agente'
)
);
}
$indexed_agents = index_array($array_agents, 'id_agente', false);
if ($id_agent === false) {
return $indexed_agents;
}
foreach ($indexed_agents as $agent => $key) {
if ($agent === $id_agent) {
return true;
}
}
return false;
}

View File

@ -455,6 +455,10 @@ function config_update_config()
$error_update[] = __('Ipam Ocuppied Manager Warning');
}
if (!config_update_value('sap_license', get_parameter('sap_license'))) {
$error_update[] = __('Deset SAP license');
}
$inventory_changes_blacklist = get_parameter('inventory_changes_blacklist', []);
if (!config_update_value('inventory_changes_blacklist', implode(',', $inventory_changes_blacklist))) {
$error_update[] = __('Inventory changes blacklist');

View File

@ -155,6 +155,8 @@ if (!empty($sub2)) {
enterprise_hook('cluster_menu');
enterprise_hook('aws_menu');
enterprise_hook('SAP_view');
if (!empty($sub)) {
$menu_operation['estado']['text'] = __('Monitoring');

View File

@ -699,7 +699,7 @@ CREATE TABLE IF NOT EXISTS `tgrupo` (
CREATE TABLE IF NOT EXISTS `tcredential_store` (
`identifier` varchar(100) NOT NULL,
`id_group` mediumint(4) unsigned NOT NULL DEFAULT 0,
`product` enum('CUSTOM', 'AWS', 'AZURE', 'GOOGLE') default 'CUSTOM',
`product` enum('CUSTOM', 'AWS', 'AZURE', 'GOOGLE', 'SAP') default 'CUSTOM',
`username` text,
`password` text,
`extra_1` text,

View File

@ -349,6 +349,12 @@ sub pandora_load_config {
$pa_config->{"fping"} = "/usr/sbin/fping"; # > 5.1SP2
# Discovery SAP
$pa_config->{"java"} = "/usr/bin/java";
# Discovery SAP utils
$pa_config->{"sap_utils"} = "/usr/share/pandora_server/util/recon_scripts/SAP";
# braa for enterprise snmp server
$pa_config->{"braa"} = "/usr/bin/braa";
@ -841,6 +847,12 @@ sub pandora_load_config {
elsif ($parametro =~ m/^fping\s(.*)/i) {
$pa_config->{'fping'}= clean_blank($1);
}
elsif ($parametro =~ m/^java\s(.*)/i) {
$pa_config->{'java'}= clean_blank($1);
}
elsif ($parametro =~ m/^sap_utils\s(.*)/i) {
$pa_config->{'sap_utils'}= clean_blank($1);
}
elsif ($parametro =~ m/^nmap_timing_template\s+([0-9]*)/i) {
$pa_config->{'nmap_timing_template'}= clean_blank($1);
}

View File

@ -56,7 +56,25 @@ my $TaskSem :shared;
use constant {
OS_OTHER => 10,
OS_ROUTER => 17,
OS_SWITCH => 18
OS_SWITCH => 18,
STEP_SCANNING => 1,
STEP_AFT => 2,
STEP_TRACEROUTE => 3,
STEP_GATEWAY => 4,
STEP_STATISTICS => 1,
STEP_APP_SCAN => 2,
STEP_CUSTOM_QUERIES => 3,
DISCOVERY_HOSTDEVICES => 0,
DISCOVERY_HOSTDEVICES_CUSTOM => 1,
DISCOVERY_CLOUD_AWS => 2,
DISCOVERY_APP_VMWARE => 3,
DISCOVERY_APP_MYSQL => 4,
DISCOVERY_APP_ORACLE => 5,
DISCOVERY_CLOUD_AWS_EC2 => 6,
DISCOVERY_CLOUD_AWS_RDS => 7,
DISCOVERY_CLOUD_AZURE_COMPUTE => 8,
DISCOVERY_DEPLOY_AGENTS => 9,
DISCOVERY_APP_SAP => 10,
};
########################################################################################
@ -196,6 +214,31 @@ sub data_consumer ($$) {
return;
}
if ($task->{'type'} == DISCOVERY_APP_SAP) {
# SAP TASK, retrieve license.
$task->{'sap_license'} = pandora_get_config_value(
$dbh,
'sap_license'
);
# Retrieve credentials for task (optional).
if (defined($task->{'auth_strings'})
&& $task->{'auth_strings'} ne ''
) {
my $key = credential_store_get_key(
$pa_config,
$dbh,
$task->{'auth_strings'}
);
# Inside an eval, here it shouln't fail unless bad configured.
$task->{'username'} = $key->{'username'};
$task->{'password'} = $key->{'password'};
}
}
my $recon = new PandoraFMS::Recon::Base(
communities => \@communities,
dbh => $dbh,

View File

@ -22,7 +22,7 @@ use constant {
STEP_TRACEROUTE => 3,
STEP_GATEWAY => 4,
STEP_STATISTICS => 1,
STEP_DATABASE_SCAN => 2,
STEP_APP_SCAN => 2,
STEP_CUSTOM_QUERIES => 3,
DISCOVERY_HOSTDEVICES => 0,
DISCOVERY_HOSTDEVICES_CUSTOM => 1,
@ -34,6 +34,7 @@ use constant {
DISCOVERY_CLOUD_AWS_RDS => 7,
DISCOVERY_CLOUD_AZURE_COMPUTE => 8,
DISCOVERY_DEPLOY_AGENTS => 9,
DISCOVERY_APP_SAP => 10,
};
# $DEVNULL
@ -1440,10 +1441,11 @@ sub cloud_scan($) {
my $type = '';
if ($self->{'task_data'}->{'type'} == DISCOVERY_CLOUD_AWS_EC2
|| $self->{'task_data'}->{'type'} == DISCOVERY_CLOUD_AWS_RDS) {
if ( $self->{'task_data'}->{'type'} == DISCOVERY_CLOUD_AWS_EC2
|| $self->{'task_data'}->{'type'} == DISCOVERY_CLOUD_AWS_RDS) {
$type = 'Aws';
} else {
# Unrecognized task type.
call('message', 'Unrecognized task type', 1);
$self->call('update_progress', -1);
@ -1465,9 +1467,11 @@ sub cloud_scan($) {
);
if (!$cloudObj) {
# Failed to initialize, check Cloud credentials or anything.
call('message', 'Unable to initialize PandoraFMS::Recon::Cloud::'.$type, 3);
} else {
# Let Cloud object manage scan.
$cloudObj->scan();
}
@ -1479,6 +1483,85 @@ sub cloud_scan($) {
}
##########################################################################
# Performs a database scan.
##########################################################################
sub database_scan($$$) {
my ($self, $type, $obj, $global_percent, $targets) = @_;
my @data;
my @modules;
my $dbObjCfg = $obj->get_config();
$self->{'summary'}->{'discovered'} += 1;
$self->{'summary'}->{'alive'} += 1;
push @modules,
{
name => $type . ' connection',
type => 'generic_proc',
data => 1,
description => $type . ' availability'
};
# Analyze.
$self->{'step'} = STEP_STATISTICS;
$self->{'c_network_percent'} = 30;
$self->call('update_progress', $global_percent + (30 / (scalar @$targets)));
$self->{'c_network_name'} = $obj->get_host();
# Retrieve connection statistics.
# Retrieve uptime statistics
# Retrieve query stats
# Retrieve connections
# Retrieve innodb
# Retrieve cache
$self->{'c_network_percent'} = 50;
$self->call('update_progress', $global_percent + (50 / (scalar @$targets)));
push @modules, $obj->get_statistics();
# Custom queries.
$self->{'step'} = STEP_CUSTOM_QUERIES;
$self->{'c_network_percent'} = 80;
$self->call('update_progress', $global_percent + (80 / (scalar @$targets)));
push @modules, $obj->execute_custom_queries();
if (defined($dbObjCfg->{'scan_databases'})
&& "$dbObjCfg->{'scan_databases'}" eq "1") {
# Skip database scan in Oracle tasks
next if $self->{'type'} == DISCOVERY_APP_ORACLE;
my $__data = $obj->scan_databases();
if (ref($__data) eq "ARRAY") {
if (defined($dbObjCfg->{'agent_per_database'})
&& $dbObjCfg->{'agent_per_database'} == 1) {
# Agent per database detected.
push @data, @{$__data};
} else {
# Merge modules into engine agent.
my @_modules = map {
map { $_ }
@{$_->{'module_data'}}
} @{$__data};
push @modules, @_modules;
}
}
}
return {
'modules' => @modules,
'data' => @data
};
}
##########################################################################
# Perform an Application scan.
##########################################################################
@ -1487,11 +1570,15 @@ sub app_scan($) {
my ($progress, $step);
my $type = '';
my $db_scan = 0;
# APP object initialization.
if ($self->{'task_data'}->{'type'} == DISCOVERY_APP_MYSQL) {
$type = 'MySQL';
} elsif ($self->{'task_data'}->{'type'} == DISCOVERY_APP_ORACLE) {
$type = 'Oracle';
} elsif ($self->{'task_data'}->{'type'} == DISCOVERY_APP_SAP) {
$type = 'SAP';
} else {
# Unrecognized task type.
call('message', 'Unrecognized task type', 1);
@ -1505,10 +1592,17 @@ sub app_scan($) {
my $global_percent = 0;
my $i = 0;
foreach my $target (@targets) {
if ( !defined($target)
|| $target eq ''
|| $target =~ /^#/) {
# Ignore empty target or commented one.
next;
}
my @data;
my @modules;
$self->{'step'} = STEP_DATABASE_SCAN;
$self->{'step'} = STEP_APP_SCAN;
$self->{'c_network_name'} = $target;
$self->{'c_network_percent'} = 0;
@ -1524,16 +1618,25 @@ sub app_scan($) {
$self->call('update_progress', $global_percent + (10 / (scalar @targets)));
# Connect to target.
my $dbObj = PandoraFMS::Recon::Util::enterprise_new(
my $obj = PandoraFMS::Recon::Util::enterprise_new(
'PandoraFMS::Recon::Applications::'.$type,
$self->{'task_data'}
{
%{$self->{'task_data'}},
'target' => $target,
'pa_config' => $self->{'pa_config'},
'parent' => $self
},
);
if (defined($dbObj)) {
if (!$dbObj->is_connected()) {
if (defined($obj)) {
# Verify if object is connected. If cannot connect to current target
# return with module.
if (!$obj->is_connected()) {
call('message', 'Cannot connect to target ' . $target, 3);
$global_percent += $global_step;
$self->{'c_network_percent'} = 90;
# Update progress
$self->call('update_progress', $global_percent + (90 / (scalar @targets)));
$self->{'summary'}->{'not_alive'} += 1;
@ -1545,74 +1648,51 @@ sub app_scan($) {
};
} else {
my $dbObjCfg = $dbObj->get_config();
#
# $results is always a hash with:
# @modules => 'global' modules.
# @data => {
# 'agent_data' => {}
# 'module_data' => []
# }
my $results;
$self->{'summary'}->{'discovered'} += 1;
$self->{'summary'}->{'alive'} += 1;
# Scan connected obj.
if ( $self->{'task_data'}->{'type'} == DISCOVERY_APP_MYSQL
|| $self->{'task_data'}->{'type'} == DISCOVERY_APP_ORACLE) {
push @modules, {
name => $type . ' connection',
type => 'generic_proc',
data => 1,
description => $type . ' availability'
};
# Database.
$results = $self->database_scan($type, $obj, $global_percent, \@targets);
# Analyze.
$self->{'step'} = STEP_STATISTICS;
$self->{'c_network_percent'} = 30;
$self->call('update_progress', $global_percent + (30 / (scalar @targets)));
$self->{'c_network_name'} = $dbObj->get_host();
} elsif ($self->{'task_data'}->{'type'} == DISCOVERY_APP_SAP) {
# Retrieve connection statistics.
# Retrieve uptime statistics
# Retrieve query stats
# Retrieve connections
# Retrieve innodb
# Retrieve cache
$self->{'c_network_percent'} = 50;
$self->call('update_progress', $global_percent + (50 / (scalar @targets)));
push @modules, $dbObj->get_statistics();
# SAP scan
$results = $obj->scan();
# Custom queries.
$self->{'step'} = STEP_CUSTOM_QUERIES;
$self->{'c_network_percent'} = 80;
$self->call('update_progress', $global_percent + (80 / (scalar @targets)));
push @modules, $dbObj->execute_custom_queries();
}
if (defined($dbObjCfg->{'scan_databases'})
&& "$dbObjCfg->{'scan_databases'}" eq "1") {
# Skip database scan in Oracle tasks
next if $self->{'type'} == DISCOVERY_APP_ORACLE;
# Add results.
if (ref($results) eq 'HASH') {
if (defined($results->{'modules'})) {
push @modules, $results->{'modules'};
}
my $__data = $dbObj->scan_databases();
if (ref($__data) eq "ARRAY") {
if (defined($dbObjCfg->{'agent_per_database'})
&& $dbObjCfg->{'agent_per_database'} == 1) {
# Agent per database detected.
push @data, @{$__data};
} else {
# Merge modules into engine agent.
my @_modules = map {
map { $_ } @{$_->{'module_data'}}
} @{$__data};
push @modules, @_modules;
}
if (defined($results->{'data'})) {
push @data, $results->{'data'};
}
}
}
# Put engine agent at the beginning of the list.
my $version = $dbObj->get_version();
unshift @data,{
my $version = $obj->get_version();
unshift @data, {
'agent_data' => {
'agent_name' => $dbObj->get_agent_name(),
'agent_name' => $obj->get_agent_name(),
'os' => $type,
'os_version' => (defined($version) ? $version : 'Discovery'),
'interval' => $self->{'task_data'}->{'interval_sweep'},
'id_group' => $self->{'task_data'}->{'id_group'},
'address' => $dbObj->get_host(),
'address' => $obj->get_host(),
'description' => '',
},
'module_data' => \@modules,
@ -1621,7 +1701,7 @@ sub app_scan($) {
$self->call('create_agents', \@data);
# Destroy item.
undef($dbObj);
undef($obj);
}
$global_percent += $global_step;
@ -1657,9 +1737,11 @@ sub deploy_scan($) {
);
if (!$deployer) {
# Failed to initialize, check Cloud credentials or anything.
call('message', 'Unable to initialize PandoraFMS::Recon::Deployer', 3);
} else {
# Let deployer object manage scan.
$deployer->scan();
}
@ -1682,13 +1764,16 @@ sub scan($) {
$self->call('update_progress', 1);
if (defined($self->{'task_data'})) {
if ($self->{'task_data'}->{'type'} == DISCOVERY_APP_MYSQL
|| $self->{'task_data'}->{'type'} == DISCOVERY_APP_ORACLE) {
# Database scan.
if ( $self->{'task_data'}->{'type'} == DISCOVERY_APP_MYSQL
|| $self->{'task_data'}->{'type'} == DISCOVERY_APP_ORACLE
|| $self->{'task_data'}->{'type'} == DISCOVERY_APP_SAP) {
# Application scan.
return $self->app_scan();
}
if ($self->{'task_data'}->{'type'} == DISCOVERY_CLOUD_AWS_RDS) {
# Cloud scan.
return $self->cloud_scan();
}

View File

@ -110,6 +110,7 @@ our @EXPORT = qw(
is_offline
to_number
clean_blank
credential_store_get_key
pandora_sendmail
pandora_trash_ascii
enterprise_hook
@ -487,6 +488,27 @@ sub pandora_daemonize {
# Pandora other General functions |
# -------------------------------------------+
########################################################################
# SUB credential_store_get_key
# Retrieve all information related to target identifier.
# param1 - config hash
# param2 - dbh link
# param3 - string identifier
########################################################################
sub credential_store_get_key($$$) {
my ($pa_config, $dbh, $identifier) = @_;
my $sql = 'SELECT * FROM tcredential_store WHERE identifier = ?';
my $key = get_db_single_row($dbh, $sql, $identifier);
return {
'username' => pandora_output_password($pa_config, $key->{'username'}),
'password' => pandora_output_password($pa_config, $key->{'password'}),
'extra_1' => $key->{'extra_1'},
'extra_2' => $key->{'extra_2'},
};
}
########################################################################
# SUB pandora_sendmail