tyler.durden
/
centreon-plugins
mirror of https://github.com/centreon/centreon-plugins.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enh prometheus nodeexporter metrics (#1179) 

* enh prometheus nodeexporter metrics handling

* add message multiple

* simplify label_replace, typos
This commit is contained in:
Colin Gagnaire 2018-10-19 09:50:35 +02:00 committed by GitHub
parent 14c2a34a9c
commit a3527417a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 186 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cloud/prometheus/exporters/nodeexporter/mode/cpu.pm
View File

@ -30,7 +30,7 @@ sub set_counters {
$self->{maps_counters_type} = [
{ name => 'nodes', type => 3, cb_prefix_output => 'prefix_nodes_output', message_multiple => 'All nodes CPU usage are ok',
counters => [ { name => 'cpu', type => 1, cb_prefix_output => 'prefix_cpu_output' } ] },
counters => [ { name => 'cpu', type => 1, cb_prefix_output => 'prefix_cpu_output', message_multiple => 'All CPU usage are ok' } ] },
];
$self->{maps_counters}->{nodes} = [
@ -82,11 +82,26 @@ sub new {
{
"node:s" => { name => 'node', default => '.*' },
"extra-filter:s@" => { name => 'extra_filter' },
"metric-overload:s@" => { name => 'metric_overload' },
});
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
$self->{metrics} = {
'cpu' => "^node_cpu.*",
};
foreach my $metric (@{$self->{option_results}->{metric_overload}}) {
next if ($metric !~ /(.*),(.*)/);
$self->{metrics}->{$1} = $2 if (defined($self->{metrics}->{$1}));
}
}
sub manage_selection {
my ($self, %options) = @_;
@ -98,7 +113,7 @@ sub manage_selection {
$extra_filter .= ',' . $filter;
}
my $results = $options{custom}->query_range(queries => [ '(1 - irate(node_cpu_seconds_total{mode="idle",instance=~"' . $self->{option_results}->{node} .
my $results = $options{custom}->query_range(queries => [ '(1 - irate({__name__=~"' . $self->{metrics}->{cpu} . '",mode="idle",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}[1m])) * 100' ]);
foreach my $metric (@{$results}) {
@ -132,11 +147,7 @@ Check CPU usage for nodes and each of their cores.
=item B<--node>
Filter on a specific node (Must be a regexp)
=item B<--extra-filter>
Set a PromQL filter (Can be multiple, Example : 'name=~".*pretty.*"')
Filter on a specific node (Must be a regexp, Default: '.*')
=item B<--warning-*>
@ -148,6 +159,18 @@ Can be: 'node-usage', 'cpu-usage'.
Threshold critical.
Can be: 'node-usage', 'cpu-usage'.
=item B<--extra-filter>
Add a PromQL filter (Can be multiple)
Example : --extra-filter='name=~".*pretty.*"'
=item B<--metric-overload>
Overload default metrics name (Can be multiple, metric can be 'cpu')
Example : --metric-overload='metric,^my_metric_name$'
=back
=cut

38
cloud/prometheus/exporters/nodeexporter/mode/cpudetailed.pm
View File

@ -30,7 +30,7 @@ sub set_counters {
$self->{maps_counters_type} = [
{ name => 'nodes', type => 3, cb_prefix_output => 'prefix_nodes_output', message_multiple => 'All nodes CPU usage are ok',
counters => [ { name => 'cpu', type => 1, cb_prefix_output => 'prefix_cpu_output' } ] },
counters => [ { name => 'cpu', type => 1, cb_prefix_output => 'prefix_cpu_output', message_multiple => 'All CPU usage are ok' } ] },
];
$self->{maps_counters}->{nodes} = [
@ -221,23 +221,37 @@ sub new {
{
"node:s" => { name => 'node', default => '.*' },
"extra-filter:s@" => { name => 'extra_filter' },
"metric-overload:s@" => { name => 'metric_overload' },
});
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
$self->{metrics} = {
'cpu' => "^node_cpu.*",
};
foreach my $metric (@{$self->{option_results}->{metric_overload}}) {
next if ($metric !~ /(.*),(.*)/);
$self->{metrics}->{$1} = $2 if (defined($self->{metrics}->{$1}));
}
}
sub manage_selection {
my ($self, %options) = @_;
$self->{nodes} = {};
$self->{cpu} = {};
my $extra_filter = '';
foreach my $filter (@{$self->{option_results}->{extra_filter}}) {
$extra_filter .= ',' . $filter;
}
my $results = $options{custom}->query_range(queries => [ '(irate(node_cpu_seconds_total{instance=~"' . $self->{option_results}->{node} .
my $results = $options{custom}->query_range(queries => [ '(irate({__name__=~"' . $self->{metrics}->{cpu} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}[1m])) * 100' ]);
foreach my $metric (@{$results}) {
@ -274,11 +288,7 @@ Check CPU detailed usage for nodes and each of their cores.
=item B<--node>
Filter on a specific node (Must be a regexp)
=item B<--extra-filter>
Set a PromQL filter (Can be multiple, Example : 'name=~".*pretty.*"')
Filter on a specific node (Must be a regexp, Default: '.*')
=item B<--warning-*>
@ -296,6 +306,18 @@ Can be: 'node-idle', 'node-wait', 'node-irq', 'node-nice',
'cpu-idle', 'cpu-wait', 'cpu-irq', 'cpu-nice', 'cpu-softirq',
'cpu-steal', 'cpu-system', 'cpu-user'.
=item B<--extra-filter>
Add a PromQL filter (Can be multiple)
Example : --extra-filter='name=~".*pretty.*"'
=item B<--metric-overload>
Overload default metrics name (Can be multiple, metric can be 'cpu')
Example : --metric-overload='metric,^my_metric_name$'
=back
=cut

62
cloud/prometheus/exporters/nodeexporter/mode/load.pm
View File

@ -34,31 +34,31 @@ sub set_counters {
$self->{maps_counters}->{nodes} = [
{ label => 'load1', set => {
key_values => [ { name => 'node_load1' }, { name => 'display' } ],
key_values => [ { name => 'load1' }, { name => 'display' } ],
output_template => 'Load 1 minute: %.2f',
output_change_bytes => 1,
perfdatas => [
{ label => 'load1', value => 'node_load1_absolute', template => '%.2f',
{ label => 'load1', value => 'load1_absolute', template => '%.2f',
min => 0, label_extra_instance => 1, instance_use => 'display_absolute' },
],
}
},
{ label => 'load5', set => {
key_values => [ { name => 'node_load5' }, { name => 'display' } ],
key_values => [ { name => 'load5' }, { name => 'display' } ],
output_template => 'Load 5 minutes: %.2f',
output_change_bytes => 1,
perfdatas => [
{ label => 'load5', value => 'node_load5_absolute', template => '%.2f',
{ label => 'load5', value => 'load5_absolute', template => '%.2f',
min => 0, label_extra_instance => 1, instance_use => 'display_absolute' },
],
}
},
{ label => 'load15', set => {
key_values => [ { name => 'node_load15' }, { name => 'display' } ],
key_values => [ { name => 'load15' }, { name => 'display' } ],
output_template => 'Load 15 minutes: %.2f',
output_change_bytes => 1,
perfdatas => [
{ label => 'load15', value => 'node_load15_absolute', template => '%.2f',
{ label => 'load15', value => 'load15_absolute', template => '%.2f',
min => 0, label_extra_instance => 1, instance_use => 'display_absolute' },
],
}
@ -82,29 +82,45 @@ sub new {
{
"node:s" => { name => 'node', default => '.*' },
"extra-filter:s@" => { name => 'extra_filter' },
"metric-overload:s@" => { name => 'metric_overload' },
});
return $self;
}
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
$self->{metrics} = {
'load1' => '^node_load1$',
'load5' => '^node_load5$',
'load15' => '^node_load15$',
};
foreach my $metric (@{$self->{option_results}->{metric_overload}}) {
next if ($metric !~ /(.*),(.*)/);
$self->{metrics}->{$1} = $2 if (defined($self->{metrics}->{$1}));
}
}
sub manage_selection {
my ($self, %options) = @_;
$self->{nodes} = {};
$self->{cpu} = {};
my $extra_filter = '';
foreach my $filter (@{$self->{option_results}->{extra_filter}}) {
$extra_filter .= ',' . $filter;
}
my $results = $options{custom}->query_range(queries => [ 'node_load1{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}',
'node_load5{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}',
'node_load15{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}' ]);
my $results = $options{custom}->query_range(queries => [ 'label_replace({__name__=~"' . $self->{metrics}->{load1} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "load1", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{load5} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "load5", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{load15} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "load15", "", "")' ]);
foreach my $metric (@{$results}) {
my $average = $options{custom}->compute(aggregation => 'average', values => $metric->{values});
$self->{nodes}->{$metric->{metric}->{instance}}->{display} = $metric->{metric}->{instance};
@ -129,11 +145,7 @@ Check nodes load.
=item B<--node>
Filter on a specific node (Must be a regexp)
=item B<--extra-filter>
Set a PromQL filter (Can be multiple, Example : 'name=~".*pretty.*"')
Filter on a specific node (Must be a regexp, Default: '.*')
=item B<--warning-*>
@ -145,6 +157,18 @@ Can be: 'load1', 'load5', 'load15'.
Threshold critical.
Can be: 'load1', 'load5', 'load15'.
=item B<--extra-filter>
Add a PromQL filter (Can be multiple)
Example : --extra-filter='name=~".*pretty.*"'
=item B<--metric-overload>
Overload default metrics name (Can be multiple, metric can be 'load1', 'load5', 'load15')
Example : --metric-overload='metric,^my_metric_name$'
=back
=cut

71
cloud/prometheus/exporters/nodeexporter/mode/memory.pm
View File

@ -75,12 +75,12 @@ sub custom_usage_calc {
my ($self, %options) = @_;
$self->{result_values}->{display} = $options{new_datas}->{$self->{instance} . '_display'};
$self->{result_values}->{total} = $options{new_datas}->{$self->{instance} . '_node_memory_MemTotal_bytes'};
$self->{result_values}->{available} = $options{new_datas}->{$self->{instance} . '_node_memory_MemAvailable_bytes'};
$self->{result_values}->{buffer} = $options{new_datas}->{$self->{instance} . '_node_memory_Buffers_bytes'};
$self->{result_values}->{cached} = $options{new_datas}->{$self->{instance} . '_node_memory_Cached_bytes'};
$self->{result_values}->{total} = $options{new_datas}->{$self->{instance} . '_total'};
$self->{result_values}->{available} = $options{new_datas}->{$self->{instance} . '_available'};
$self->{result_values}->{buffer} = $options{new_datas}->{$self->{instance} . '_buffer'};
$self->{result_values}->{cached} = $options{new_datas}->{$self->{instance} . '_cached'};
$self->{result_values}->{used} = $self->{result_values}->{total} - $self->{result_values}->{available} - $self->{result_values}->{buffer} - $self->{result_values}->{cached};
$self->{result_values}->{prct_used} = $self->{result_values}->{used} * 100 / ($self->{result_values}->{total});
$self->{result_values}->{prct_used} = ($self->{result_values}->{total} > 0) ? $self->{result_values}->{used} * 100 / $self->{result_values}->{total} : 0;
return 0;
}
@ -94,8 +94,8 @@ sub set_counters {
$self->{maps_counters}->{nodes} = [
{ label => 'usage', set => {
key_values => [ { name => 'node_memory_MemTotal_bytes' }, { name => 'node_memory_MemAvailable_bytes' },
{ name => 'node_memory_Buffers_bytes' }, { name => 'node_memory_Cached_bytes' }, { name => 'display' } ],
key_values => [ { name => 'total' }, { name => 'available' },
{ name => 'buffer' }, { name => 'cached' }, { name => 'display' } ],
closure_custom_calc => $self->can('custom_usage_calc'),
closure_custom_output => $self->can('custom_usage_output'),
closure_custom_perfdata => $self->can('custom_usage_perfdata'),
@ -103,21 +103,21 @@ sub set_counters {
}
},
{ label => 'buffer', set => {
key_values => [ { name => 'node_memory_Buffers_bytes' }, { name => 'display' } ],
key_values => [ { name => 'buffer' }, { name => 'display' } ],
output_template => 'Buffer: %.2f %s',
output_change_bytes => 1,
perfdatas => [
{ label => 'buffer', value => 'node_memory_Buffers_bytes_absolute', template => '%s',
{ label => 'buffer', value => 'buffer_absolute', template => '%s',
min => 0, unit => 'B', label_extra_instance => 1, instance_use => 'display_absolute' },
],
}
},
{ label => 'cached', set => {
key_values => [ { name => 'node_memory_Cached_bytes' }, { name => 'display' } ],
key_values => [ { name => 'cached' }, { name => 'display' } ],
output_template => 'Cached: %.2f %s',
output_change_bytes => 1,
perfdatas => [
{ label => 'cached', value => 'node_memory_Cached_bytes_absolute', template => '%s',
{ label => 'cached', value => 'cached_absolute', template => '%s',
min => 0, unit => 'B', label_extra_instance => 1, instance_use => 'display_absolute' },
],
}
@ -142,6 +142,7 @@ sub new {
"node:s" => { name => 'node', default => '.*' },
"extra-filter:s@" => { name => 'extra_filter' },
"units:s" => { name => 'units', default => '%' },
"metric-overload:s@" => { name => 'metric_overload' },
});
return $self;
@ -151,6 +152,18 @@ sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
$self->{metrics} = {
'total' => "^node_memory_MemTotal.*",
'available' => "^node_memory_MemAvailable.*",
'cached' => "^node_memory_Cached.*",
'buffer' => "^node_memory_Buffers.*",
};
foreach my $metric (@{$self->{option_results}->{metric_overload}}) {
next if ($metric !~ /(.*),(.*)/);
$self->{metrics}->{$1} = $2 if (defined($self->{metrics}->{$1}));
}
$instance_mode = $self;
}
@ -164,15 +177,15 @@ sub manage_selection {
$extra_filter .= ',' . $filter;
}
my $results = $options{custom}->query_range(queries => [ 'node_memory_MemTotal_bytes{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}',
'node_memory_MemAvailable_bytes{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}',
'node_memory_Cached_bytes{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}',
'node_memory_Buffers_bytes{instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}' ]);
my $results = $options{custom}->query_range(queries => [ 'label_replace({__name__=~"' . $self->{metrics}->{total} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "total", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{available} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "available", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{cached} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "cached", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{buffer} . '",instance=~"' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "buffer", "", "")' ]);
foreach my $metric (@{$results}) {
my $average = $options{custom}->compute(aggregation => 'average', values => $metric->{values});
$self->{nodes}->{$metric->{metric}->{instance}}->{display} = $metric->{metric}->{instance};
@ -197,11 +210,7 @@ Check memory usage.
=item B<--node>
Filter on a specific node (Must be a regexp)
=item B<--extra-filter>
Set a PromQL filter (Can be multiple, Example : 'name=~".*pretty.*"')
Filter on a specific node (Must be a regexp, Default: '.*')
=item B<--warning-*>
@ -213,6 +222,18 @@ Can be: 'usage', 'buffer', 'cached'.
Threshold critical.
Can be: 'usage', 'buffer', 'cached'.
=item B<--extra-filter>
Add a PromQL filter (Can be multiple)
Example : --extra-filter='name=~".*pretty.*"'
=item B<--metric-overload>
Overload default metrics name (Can be multiple, metric can be 'total', 'available', 'cached', 'buffer')
Example : --metric-overload='metric,^my_metric_name$'
=back
=cut

50
cloud/prometheus/exporters/nodeexporter/mode/storage.pm
View File

@ -84,10 +84,10 @@ sub custom_usage_calc {
$self->{result_values}->{display} = $options{new_datas}->{$self->{instance} . '_display'};
$self->{result_values}->{multi} = $options{new_datas}->{$self->{instance} . '_multi'};
$self->{result_values}->{total} = $options{new_datas}->{$self->{instance} . '_node_filesystem_size_bytes'};
$self->{result_values}->{free} = $options{new_datas}->{$self->{instance} . '_node_filesystem_free_bytes'};
$self->{result_values}->{total} = $options{new_datas}->{$self->{instance} . '_size'};
$self->{result_values}->{free} = $options{new_datas}->{$self->{instance} . '_free'};
$self->{result_values}->{used} = $self->{result_values}->{total} - $self->{result_values}->{free};
$self->{result_values}->{prct_used} = $self->{result_values}->{used} * 100 / ($self->{result_values}->{total});
$self->{result_values}->{prct_used} = ($self->{result_values}->{total} > 0) ? $self->{result_values}->{used} * 100 / $self->{result_values}->{total} : 0;
$self->{result_values}->{prct_free} = 100 - $self->{result_values}->{prct_used};
# limit to 100. Better output.
@ -105,12 +105,12 @@ sub set_counters {
$self->{maps_counters_type} = [
{ name => 'nodes', type => 3, cb_prefix_output => 'prefix_nodes_output', message_multiple => 'All nodes storages usage are ok',
counters => [ { name => 'storage', type => 1, cb_prefix_output => 'prefix_storage_output' } ] },
counters => [ { name => 'storage', type => 1, cb_prefix_output => 'prefix_storage_output', message_multiple => 'All storages usage are ok' } ] },
];
$self->{maps_counters}->{storage} = [
{ label => 'usage', set => {
key_values => [ { name => 'node_filesystem_free_bytes' }, { name => 'node_filesystem_size_bytes' }, { name => 'multi' }, { name => 'display' } ],
key_values => [ { name => 'free' }, { name => 'size' }, { name => 'multi' }, { name => 'display' } ],
closure_custom_calc => $self->can('custom_usage_calc'),
closure_custom_output => $self->can('custom_usage_output'),
closure_custom_perfdata => $self->can('custom_usage_perfdata'),
@ -146,6 +146,7 @@ sub new {
"extra-filter:s@" => { name => 'extra_filter' },
"units:s" => { name => 'units', default => '%' },
"free" => { name => 'free' },
"metric-overload:s@" => { name => 'metric_overload' },
});
return $self;
@ -154,7 +155,17 @@ sub new {
sub check_options {
my ($self, %options) = @_;
$self->SUPER::check_options(%options);
$self->{metrics} = {
'free' => "^node_filesystem_free.*",
'size' => "^node_filesystem_size.*",
};
foreach my $metric (@{$self->{option_results}->{metric_overload}}) {
next if ($metric !~ /(.*),(.*)/);
$self->{metrics}->{$1} = $2 if (defined($self->{metrics}->{$1}));
}
$instance_mode = $self;
}
@ -162,21 +173,20 @@ sub manage_selection {
my ($self, %options) = @_;
$self->{nodes} = {};
$self->{cpu} = {};
my $extra_filter = '';
foreach my $filter (@{$self->{option_results}->{extra_filter}}) {
$extra_filter .= ',' . $filter;
}
my $results = $options{custom}->query(queries => [ 'node_filesystem_free_bytes{instance=~"' . $self->{option_results}->{node} .
my $results = $options{custom}->query(queries => [ 'label_replace({__name__=~"' . $self->{metrics}->{free} . '",instance=~"' . $self->{option_results}->{node} .
'",mountpoint=~"' . $self->{option_results}->{storage} .
'",fstype!~"' . $self->{option_results}->{exclude_storage_type} .
'"' . $extra_filter . '}',
'node_filesystem_size_bytes{instance=~".*' . $self->{option_results}->{node} .
'"' . $extra_filter . '}, "__name__", "free", "", "")',
'label_replace({__name__=~"' . $self->{metrics}->{size} . '",instance=~"' . $self->{option_results}->{node} .
'",mountpoint=~"' . $self->{option_results}->{storage} .
'",fstype!~"' . $self->{option_results}->{exclude_storage_type} .
'"' . $extra_filter . '}' ]);
'"' . $extra_filter . '}, "__name__", "size", "", "")' ]);
foreach my $metric (@{$results}) {
$self->{nodes}->{$metric->{metric}->{instance}}->{display} = $metric->{metric}->{instance};
@ -203,20 +213,16 @@ Check CPU usage for nodes and each of their cores.
=item B<--node>
Filter on a specific node (Must be a regexp)
Filter on a specific node (Must be a regexp, Default: '.*')
=item B<--storage>
Filter on a specific storage (Must be a regexp)
Filter on a specific storage (Must be a regexp, Default: '.*')
=item B<--exclude-storage-type>
Exclude storage types (Must be a regexp, Default: 'rootfs|tmpfs')
=item B<--extra-filter>
Set a PromQL filter (Can be multiple, Example : 'name=~".*pretty.*"')
=item B<--units>
Units of thresholds (Default: '%') ('%', 'B').
@ -233,6 +239,18 @@ Threshold warning.
Threshold critical.
=item B<--extra-filter>
Add a PromQL filter (Can be multiple)
Example : --extra-filter='name=~".*pretty.*"'
=item B<--metric-overload>
Overload default metrics name (Can be multiple, metric can be 'free', 'size')
Example : --metric-overload='metric,^my_metric_name$'
=back
=cut

3
cloud/prometheus/restapi/custom/api.pm
View File

@ -176,6 +176,8 @@ sub query_range {
my $uri = URI::Encode->new({encode_reserved => 1});
foreach my $query (@{$options{queries}}) {
$self->{output}->output_add(long_msg => sprintf("Query range: '/query_range?query=%s&start=%s&end=%s&step=%s'",
$query, $start_time, $end_time, $self->{step}), debug => 1);
my $result = $self->get_endpoint(url_path => '/query_range?query=' . $uri->encode($query) .
'&start=' . $start_time . '&end=' . $end_time . '&step=' . $self->{step});
push @{$data}, @{$result->{result}};
@ -191,6 +193,7 @@ sub query {
my $uri = URI::Encode->new({encode_reserved => 1});
foreach my $query (@{$options{queries}}) {
$self->{output}->output_add(long_msg => sprintf("Query: '/query?query=%s'", $query), debug => 1);
my $result = $self->get_endpoint(url_path => '/query?query=' . $uri->encode($query));
push @{$data}, @{$result->{result}};
}