(plugin) apps::monitoring::splunk - adding plugin to monitor Splunk (#3927)

* draft for splunk plugin

* wip

* adding draft index mode

* enh

* enh help

* Apply suggestion

* some minor fixes

* some minor fixes

* some minor fixes

* some minor fixes

Co-authored-by: garnier-quentin <garnier.quentin@gmail.com>
This commit is contained in:
lchrdn 2022-10-06 11:46:10 +02:00 committed by GitHub
parent db879e3356
commit 5728d2eca0
5 changed files with 808 additions and 0 deletions

View File

@ -0,0 +1,396 @@
#
# Copyright 2022 Centreon (http://www.centreon.com/)
#
# Centreon is a full-fledged industry-strength solution that meets
# the needs in IT infrastructure and application monitoring for
# service performance.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
package apps::monitoring::splunk::custom::api;
use strict;
use warnings;
use centreon::plugins::http;
use centreon::plugins::statefile;
use DateTime;
use XML::LibXML::Simple;
use URI::Encode;
use Digest::MD5 qw(md5_hex);
sub new {
my ($class, %options) = @_;
my $self = {};
bless $self, $class;
if (!defined($options{output})) {
print "Class Custom: Need to specify 'output' argument.\n";
exit 3;
}
if (!defined($options{options})) {
$options{output}->add_option_msg(short_msg => "Class Custom: Need to specify 'options' argument.");
$options{output}->option_exit();
}
if (!defined($options{noptions})) {
$options{options}->add_options(arguments => {
'hostname:s' => { name => 'hostname' },
'url-path:s' => { name => 'url_path' },
'port:s' => { name => 'port' },
'proto:s' => { name => 'proto' },
'api-username:s' => { name => 'api_username' },
'api-password:s' => { name => 'api_password' },
'timeout:s' => { name => 'timeout' },
'unknown-http-status:s' => { name => 'unknown_http_status' },
'warning-http-status:s' => { name => 'warning_http_status' },
'critical-http-status:s' => { name => 'critical_http_status' }
});
}
$options{options}->add_help(package => __PACKAGE__, sections => 'XMLAPI OPTIONS', once => 1);
$self->{output} = $options{output};
$self->{http} = centreon::plugins::http->new(%options);
$self->{cache} = centreon::plugins::statefile->new(%options);
return $self;
}
sub set_options {
my ($self, %options) = @_;
$self->{option_results} = $options{option_results};
}
sub set_defaults {}
sub check_options {
my ($self, %options) = @_;
$self->{hostname} = (defined($self->{option_results}->{hostname})) ? $self->{option_results}->{hostname} : '';
$self->{port} = (defined($self->{option_results}->{port})) ? $self->{option_results}->{port} : 8089;
$self->{proto} = (defined($self->{option_results}->{proto})) ? $self->{option_results}->{proto} : 'https';
$self->{timeout} = (defined($self->{option_results}->{timeout})) ? $self->{option_results}->{timeout} : 10;
$self->{api_username} = (defined($self->{option_results}->{api_username})) ? $self->{option_results}->{api_username} : '';
$self->{api_password} = (defined($self->{option_results}->{api_password})) ? $self->{option_results}->{api_password} : '';
$self->{unknown_http_status} = (defined($self->{option_results}->{unknown_http_status})) ? $self->{option_results}->{unknown_http_status} : '%{http_code} < 200 or %{http_code} >= 300';
$self->{warning_http_status} = (defined($self->{option_results}->{warning_http_status})) ? $self->{option_results}->{warning_http_status} : '';
$self->{critical_http_status} = (defined($self->{option_results}->{critical_http_status})) ? $self->{option_results}->{critical_http_status} : '';
if ($self->{hostname} eq '') {
$self->{output}->add_option_msg(short_msg => 'Need to specify hostname option.');
$self->{output}->option_exit();
}
if ($self->{api_username} eq '') {
$self->{output}->add_option_msg(short_msg => 'Need to specify --api-username option.');
$self->{output}->option_exit();
}
if ($self->{api_password} eq '') {
$self->{output}->add_option_msg(short_msg => 'Need to specify --api-password option.');
$self->{output}->option_exit();
}
$self->{cache}->check_options(option_results => $self->{option_results});
return 0;
}
sub build_options_for_httplib {
my ($self, %options) = @_;
$self->{option_results}->{hostname} = $self->{hostname};
$self->{option_results}->{timeout} = $self->{timeout};
$self->{option_results}->{port} = $self->{port};
$self->{option_results}->{proto} = $self->{proto};
}
sub clean_session_token {
my ($self, %options) = @_;
my $datas = {};
$options{statefile}->write(data => $datas);
$self->{http}->remove_header(key => 'Authorization');
$self->{session_token} = undef;
}
sub convert_iso8601_to_epoch {
my ($self, %options) = @_;
if ($options{time_string} =~ /(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})([+-]\d{4})/) {
my $dt = DateTime->new(
year => $1,
month => $2,
day => $3,
hour => $4,
minute => $5,
second => $6,
time_zone => $7
);
my $epoch_time = $dt->epoch();
return $epoch_time;
}
$self->{output}->add_option_msg(short_msg => "Wrong date format: $options{time_string}");
$self->{output}->option_exit();
}
sub settings {
my ($self, %options) = @_;
$self->build_options_for_httplib();
$self->{http}->add_header(key => 'X-Requested-By', value => $self->{requested_by});
$self->{http}->add_header(key => 'Authorization', value => 'Splunk ' . $self->{session_token}) if defined($self->{session_token});
$self->{http}->set_options(%{$self->{option_results}});
}
sub get_access_token {
my ($self, %options) = @_;
$self->settings();
my $has_cache_file = $options{statefile}->read(statefile => 'splunk_api_' . md5_hex($self->{option_results}->{hostname}) . '_' . md5_hex($self->{option_results}->{api_username}));
my $session_token = $options{statefile}->get(name => 'session_token');
if ($has_cache_file == 0 || !defined($session_token)) {
my $credentials = [
'username=' . $self->{api_username},
'password=' . $self->{api_password}
];
my ($content) = $self->{http}->request(
method => 'POST',
url_path => '/services/auth/login',
post_param => $credentials,
unknown_status => $self->{unknown_http_status},
warning_status => $self->{warning_http_status},
critical_status => $self->{critical_http_status}
);
if ($self->{http}->get_code() != 200) {
$self->{output}->add_option_msg(short_msg => "login error [code: '" . $self->{http}->get_code() . "'] [message: '" . $self->{http}->get_message() . "']");
$self->{output}->option_exit();
}
my $xml_result;
eval {
$SIG{__WARN__} = sub {};
$xml_result = XMLin($content, ForceArray => $options{force_array}, KeyAttr => []);
};
if ($@) {
$self->{output}->add_option_msg(short_msg => "Cannot decode xml response: $@");
$self->{output}->option_exit();
}
if (defined($xml_result->{XPathError})) {
$self->{output}->add_option_msg(short_msg => "Api return error: " . $xml_result->{XPathError}->{Reason});
$self->{output}->option_exit();
}
if (!defined($xml_result) || !defined($xml_result->{sessionKey})) {
$self->{output}->add_option_msg(short_msg => 'error retrieving session_token');
$self->{output}->option_exit();
}
$session_token = $xml_result->{sessionKey};
my $datas = { session_token => $session_token };
$options{statefile}->write(data => $datas);
}
return $session_token;
}
sub get_index_info {
my ($self, %options) = @_;
my $index_res_info = $self->request_api(
method => 'GET',
endpoint => '/services/data/indexes',
get_param => ['count=-1']
);
my @index_update_time;
foreach (@{$index_res_info->{entry}}){
next if (defined($_->{title}) && defined($options{index_name}) && $options{index_name} ne '' && $_->{title} !~ /$options{index_name}/);
foreach my $attribute (@{$_->{content}->{'s:dict'}->{'s:key'}}){
next if ($attribute->{name} ne 'maxTime' || !defined($attribute->{content}));
my $epoch_time = ( time() - $self->convert_iso8601_to_epoch(time_string => $attribute->{content}) );
push @index_update_time, { index_name => $_->{title}, ts_last_update => $epoch_time }
}
}
return \@index_update_time;
}
sub get_splunkd_health {
my ($self, %options) = @_;
my $splunkd_health_details = $self->request_api(
method => 'GET',
endpoint => '/services/server/health/splunkd/details',
);
my @splunkd_features_health;
foreach (@{$splunkd_health_details->{entry}->{content}->{'s:dict'}->{'s:key'}[1]->{'s:dict'}->{'s:key'}}){
my $feature_name = $_->{name};
$feature_name =~ s/ /-/g;
foreach my $sub_feature (@{$_->{'s:dict'}->{'s:key'}}){
next if $sub_feature->{name} ne 'health';
push @splunkd_features_health, { feature_name => lc($feature_name), global_health => $sub_feature->{content} }
}
}
return \@splunkd_features_health;
}
sub query_count {
my ($self, %options) = @_;
my $query_sid = $self->request_api(
method => 'POST',
endpoint => '/services/search/jobs',
post_param => [
'search=' . $options{query} . '| stats count'
]
);
if (!defined($query_sid->{sid}) || $query_sid->{sid} eq ''){
$self->{output}->add_option_msg(short_msg => "Error during process. No SID where returned after query was made. Please check your query and splunkd health.");
$self->{output}->option_exit();
}
sleep(1.5);
my $query_status = $self->request_api(
method => 'GET',
endpoint => '/services/search/jobs/' . $query_sid->{sid},
);
foreach (@{$query_status->{content}->{'s:dict'}->{'s:key'}}) {
if ($_->{name} eq 'isDone' && $_->{content} == 0){
$self->{output}->add_option_msg(short_msg => "Search command wasn't completed.");
$self->{output}->option_exit();
} elsif ($_->{name} eq 'isFailed' && $_->{content} == 1) {
$self->{output}->add_option_msg(short_msg => "Search command failed.");
$self->{output}->option_exit();
}
}
my $query_res = $self->request_api(
method => 'GET',
endpoint => '/services/search/jobs/' . $query_sid->{sid} . '/results',
);
my $query_count = $query_res->{result}->{field}->{value}->{text};
return $query_count;
}
sub request_api {
my ($self, %options) = @_;
if (!defined($self->{session_token})) {
$self->{session_token} = $self->get_access_token(statefile => $self->{cache});
}
$self->settings();
my $content = $self->{http}->request(
method => $options{method},
url_path => $options{endpoint},
get_param => $options{get_param},
post_param => $options{post_param},
unknown_status => '',
warning_status => '',
critical_status => ''
);
if ($self->{http}->get_code() < 200 || $self->{http}->get_code() >= 300) {
$self->clean_session_token(statefile => $self->{cache});
$self->get_access_token(statefile => $self->{cache});
$self->settings();
$content = $self->{http}->request(
method => $options{method},
url_path => $options{endpoint},
post_param => $options{post_param},
unknown_status => $self->{unknown_http_status},
warning_status => $self->{warning_http_status},
critical_status => $self->{critical_http_status}
);
}
my $xml_result;
eval {
$SIG{__WARN__} = sub {};
$xml_result = XMLin($content, ForceArray => $options{force_array}, KeyAttr => []);
};
if ($@) {
$self->{output}->add_option_msg(short_msg => "Cannot decode xml response: $@");
$self->{output}->option_exit();
}
if (defined($xml_result->{XPathError})) {
$self->{output}->add_option_msg(short_msg => "Api return error: " . $xml_result->{XPathError}->{Reason});
$self->{output}->option_exit();
}
return $xml_result;
}
1;
__END__
=head1 NAME
Splunk API.
=head1 SYNOPSIS
Splunk API custom mode.
=head1 XMLAPI OPTIONS
Splunk API.
=over 8
=item B<--hostname>
Splunk server address.
=item B<--port>
API port (Default: 8089)
=item B<--proto>
Specify http if needed (Default: 'https')
=item B<--api-username>
Specify api username.
=item B<--api-password>
Specify api password.
=item B<--timeout>
Set HTTP timeout.
=back
=head1 DESCRIPTION
B<custom>.
=cut

View File

@ -0,0 +1,117 @@
#
# Copyright 2022 Centreon (http://www.centreon.com/)
#
# Centreon is a full-fledged industry-strength solution that meets
# the needs in IT infrastructure and application monitoring for
# service performance.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
package apps::monitoring::splunk::mode::indexupdate;
use base qw(centreon::plugins::templates::counter);
use strict;
use warnings;
sub set_counters {
my ($self, %options) = @_;
$self->{maps_counters_type} = [
{ name => 'index', type => 1, cb_prefix_output => 'index_prefix_output', message_multiple => 'All indexes are OK' }
];
$self->{maps_counters}->{index} = [
{ label => 'index-last-update-seconds', nlabel => 'splunk.index.last.updated.seconds', set => {
key_values => [ { name => 'index_last_update' }, { name => 'expires_human' } ],
output_use => 'expires_human',
output_template => 'last update %s ',
perfdatas => [
{ template => '%s', min => 0, unit => 's', label_extra_instance => 1 }
]
}
}
];
}
sub index_prefix_output {
my ($self, %options) = @_;
return sprintf( "Index '%s' ", $options{instance});
}
sub new {
my ($class, %options) = @_;
my $self = $class->SUPER::new(package => __PACKAGE__, %options, force_new_perfdata => 1);
bless $self, $class;
$options{options}->add_options(arguments => {
'index-name:s' => { name => 'index_name' }
});
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
}
sub manage_selection {
my ($self, %options) = @_;
my $index_info = $options{custom}->get_index_info(
index_name => $self->{option_results}->{index_name}
);
foreach my $value (@{$index_info}){
$self->{index}->{$value->{index_name}} = {
index_last_update => $value->{ts_last_update},
expires_human => centreon::plugins::misc::change_seconds(value => $value->{ts_last_update})
}
}
}
1;
__END__
=head1 MODE
Check Splunk index last update time.
=over 8
=item B<--index-name>
Specify index name to get last updates.
If not specified, all indexes are checked by default.
=item B<--warning-index-last-update-seconds>
Warning threshold in seconds for last update.
Example: --warning-index-last-update-seconds=15000
=item B<--critical-index-last-update-seconds>
Critical threshold in seconds for last update.
Example: --critical-index-last-update-seconds=25000
=back
=cut

View File

@ -0,0 +1,113 @@
#
# Copyright 2022 Centreon (http://www.centreon.com/)
#
# Centreon is a full-fledged industry-strength solution that meets
# the needs in IT infrastructure and application monitoring for
# service performance.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
package apps::monitoring::splunk::mode::query;
use base qw(centreon::plugins::templates::counter);
use strict;
use warnings;
sub set_counters {
my ($self, %options) = @_;
$self->{maps_counters_type} = [
{ name => 'global', type => 0, skipped_code => { -10 => 1 } }
];
$self->{maps_counters}->{global} = [
{ label => 'query-matches', nlabel => 'splunk.query.matches.count', set => {
key_values => [ { name => 'query_matches' } ],
output_template => 'query matches: %s',
perfdatas => [
{ template => '%d', min => 0 }
]
}
}
];
}
sub new {
my ($class, %options) = @_;
my $self = $class->SUPER::new(package => __PACKAGE__, %options, force_new_perfdata => 1);
bless $self, $class;
$options{options}->add_options(arguments => {
'query:s' => { name => 'query' }
});
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
if (!defined($self->{option_results}->{query}) || $self->{option_results}->{query} eq '') {
$self->{output}->add_option_msg(short_msg => 'Please set --query option.');
$self->{output}->option_exit();
}
}
sub manage_selection {
my ($self, %options) = @_;
my $query_count = $options{custom}->query_count(
query => $self->{option_results}->{query},
timeframe => $self->{option_results}->{timeframe}
);
$self->{global} = {
query_matches => $query_count
};
}
1;
__END__
=head1 MODE
Check number of results for a query.
=over 8
=item B<--query>
Specify a query to be made and check matching number.
Query has to start with "search ".
Example: --query='search host="prod-server" "ERROR" earliest=-150000min'
=item B<--warning-query-matches>
Warning threshold for query matches.
Example: --warning-query-matches=5
=item B<--critical-query-matches>
Critical threshold for query matches.
Example: --critical-query-matches=15
=back
=cut

View File

@ -0,0 +1,131 @@
#
# Copyright 2022 Centreon (http://www.centreon.com/)
#
# Centreon is a full-fledged industry-strength solution that meets
# the needs in IT infrastructure and application monitoring for
# service performance.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
package apps::monitoring::splunk::mode::splunkdhealth;
use base qw(centreon::plugins::templates::counter);
use strict;
use warnings;
use centreon::plugins::templates::catalog_functions qw(catalog_status_threshold_ng);
sub features_long_output {
my ($self, %options) = @_;
return 'checking features';
}
sub custom_status_output {
my ($self, %options) = @_;
return "Feature '". $self->{result_values}->{display} . "'" . " health: '" . $self->{result_values}->{status} . "'";
}
sub set_counters {
my ($self, %options) = @_;
$self->{maps_counters_type} = [
{ name => 'features', type => 3, cb_long_output => 'features_long_output', indent_long_output => ' ', message_multiple => 'All features are OK.',
group => [
{ name => 'file-monitor-input', type => 0, display_short => 0, skipped_code => { -10 => 1 } },
{ name => 'index-processor', type => 0, display_short => 0, skipped_code => { -10 => 1 } },
{ name => 'resource-usage', type => 0, display_short => 0, skipped_code => { -10 => 1 } },
{ name => 'search-scheduler', type => 0, display_short => 0, skipped_code => { -10 => 1 } },
{ name => 'workload-management', type => 0, display_short => 0, skipped_code => { -10 => 1 } }
]
}
];
foreach ('file-monitor-input', 'index-processor', 'resource-usage', 'search-scheduler', 'workload-management') {
$self->{maps_counters}->{$_} = [
{
label => $_ . '-status', type => 2,
critical_default => '%{status} =~ /red/' ,
warning_default => '%{status} =~ /yellow/',
unknown_default => '%{status} !~ /(green|yellow|red)/',
set => {
key_values => [ { name => 'status' }, { name => 'display' }],
closure_custom_output => $self->can('custom_status_output'),
closure_custom_perfdata => sub { return 0; },
closure_custom_threshold_check => \&catalog_status_threshold_ng
}
}
];
}
}
sub new {
my ($class, %options) = @_;
my $self = $class->SUPER::new(package => __PACKAGE__, %options, force_new_perfdata => 1);
bless $self, $class;
$options{options}->add_options(arguments => { });
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
}
sub manage_selection {
my ($self, %options) = @_;
my $splunkd_health_details = $options{custom}->get_splunkd_health();
foreach my $feature (@{$splunkd_health_details}){
$self->{features}->{global}->{ $feature->{feature_name} } = {
display => $feature->{feature_name},
status => $feature->{global_health}
};
}
}
1;
__END__
=head1 MODE
Check the overall health of splunkd. The health of splunkd is based on the health of all features reporting it.
=over 8
=item B<--warning-*>
Warning thresholds for features status. (Default: '%{status} =~ /yellow/').
Can be: 'file-monitor-input-status', 'index-processor-status',
'resource-usage-status', 'search-scheduler-status', 'workload-management-status'
=item B<--critical-*>
Critical thresholds for features status. (Default: '%{status} =~ /red/').
Can be: 'file-monitor-input-status', 'index-processor-status',
'resource-usage-status', 'search-scheduler-status', 'workload-management-status'
=back
=cut

View File

@ -0,0 +1,51 @@
#
# Copyright 2022 Centreon (http://www.centreon.com/)
#
# Centreon is a full-fledged industry-strength solution that meets
# the needs in IT infrastructure and application monitoring for
# service performance.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
package apps::monitoring::splunk::plugin;
use strict;
use warnings;
use base qw(centreon::plugins::script_custom);
sub new {
my ($class, %options) = @_;
my $self = $class->SUPER::new(package => __PACKAGE__, %options);
bless $self, $class;
$self->{version} = '0.1';
$self->{modes} = {
'index-update' => 'apps::monitoring::splunk::mode::indexupdate',
'query' => 'apps::monitoring::splunk::mode::query',
'splunkd-health' => 'apps::monitoring::splunk::mode::splunkdhealth'
};
$self->{custom_modes}->{api} = 'apps::monitoring::splunk::custom::api';
return $self;
}
1;
__END__
=head1 PLUGIN DESCRIPTION
Check Splunk server.
=cut