tyler.durden
/
pandorafms
mirror of https://github.com/pandorafms/pandorafms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#12191 optimized command netflow and sflow

This commit is contained in:
Daniel Cebrian 2023-10-18 19:02:27 +02:00
parent 8d7f253e91
commit 8aa1e88317
4 changed files with 278 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
pandora_console/include/functions_netflow.php
View File

@ -498,7 +498,7 @@ function netflow_get_top_N(
$options = '-o "fmt:%sap,%dap,%ibyt,%bps" -q -n '.$max.' -s record/bytes -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command = netflow_get_command($options, $filter);
$command = netflow_get_command($options, $filter, $start_date, $end_date);
// Execute nfdump.
exec($command, $lines);
@ -656,7 +656,10 @@ function netflow_get_data(
$aggregate,
$max,
$absolute,
$connection_name
$connection_name,
false,
$start_date,
$end_date
);
foreach ($data as $line) {
@ -734,6 +737,8 @@ function netflow_get_data(
* to get troughput.
* @param string $connection_name Node name when data is get in meta.
* @param boolean $address_resolution True to resolve ips to hostnames.
* @param integer $start_date_fixed Start date for use in command netflow.
* @param integer $end_date_fixed End date for use in command netflow.
*
* @return array With netflow stats.
*/
@ -745,7 +750,9 @@ function netflow_get_stats(
$max,
$absolute=true,
$connection_name='',
$address_resolution=false
$address_resolution=false,
$start_date_fixed=0,
$end_date_fixed=0,
) {
global $config, $nfdump_date_format;
@ -757,8 +764,7 @@ function netflow_get_stats(
// Get the command to call nfdump.
$options = "-o csv -q -n $max -s $aggregate/bytes -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command = netflow_get_command($options, $filter);
$command = netflow_get_command($options, $filter, $start_date_fixed, $end_date_fixed);
// Execute nfdump.
exec($command, $string);
@ -845,7 +851,7 @@ function netflow_get_summary($start_date, $end_date, $filter, $connection_name='
// Get the command to call nfdump.
$options = '-o csv -n 1 -s srcip/bytes -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command = netflow_get_command($options, $filter);
$command = netflow_get_command($options, $filter, $start_date, $end_date);
// Execute nfdump.
exec($command, $string);
@ -916,7 +922,7 @@ function netflow_get_relationships_raw_data(
// Get the command to call nfdump.
$options = ' -q -o csv -n 10000 -s record/bytes -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command = netflow_get_command($options, $filter);
$command = netflow_get_command($options, $filter, $start_date, $end_date);
// Execute nfdump.
// $command .= ' -q -o csv -n 10000 -s record/bytes -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
@ -1018,7 +1024,7 @@ function netflow_parse_relationships_for_circular_mesh(
*
* @return string Command to run.
*/
function netflow_get_command($options, $filter)
function netflow_get_command($options, $filter, $date_init=0, $date_end=0)
{
global $config;
@ -1030,14 +1036,46 @@ function netflow_get_command($options, $filter)
&& isset($config['netflow_name_dir']) && $config['netflow_name_dir'] !== ''
&& isset($config['general_network_path']) && $config['general_network_path'] !== ''
) {
$command .= ' -R. -M '.$config['general_network_path'].$config['netflow_name_dir'].':'.$config['sflow_name_dir'];
if ($date_init > 0 && $date_end > 0) {
$path = $config['general_network_path'].$config['sflow_name_dir'];
$range_time = find_range_files_time($path, $date_init, $date_end);
if ($range_time[0] === 0 && $range_time[1] === 0) {
$interval_files_sflow = $path;
} else {
$interval_files_sflow = $path.'/'.$range_time[0].':'.$range_time[1];
}
$path = $config['general_network_path'].$config['netflow_name_dir'];
$range_time = find_range_files_time($path, $date_init, $date_end);
if ($range_time[0] === 0 && $range_time[1] === 0) {
$interval_files_netflow = $path;
} else {
$interval_files_netflow = $path.'/'.$range_time[0].':'.$range_time[1];
}
$command .= ' -R '.$interval_files_sflow.' -R '.$interval_files_netflow;
} else {
$command .= ' -R. -M '.$config['general_network_path'].$config['netflow_name_dir'].':'.$config['sflow_name_dir'];
}
}
} else {
if ($config['activate_sflow']) {
if (isset($config['sflow_name_dir']) && $config['sflow_name_dir'] !== ''
&& isset($config['general_network_path']) && $config['general_network_path'] !== ''
) {
$command .= ' -R. -M '.$config['general_network_path'].$config['sflow_name_dir'];
if ($date_init > 0 && $date_end > 0) {
$path = $config['general_network_path'].$config['sflow_name_dir'];
$range_time = find_range_files_time($path, $date_init, $date_end);
if ($range_time[0] === 0 && $range_time[1] === 0) {
$interval_files = '.';
} else {
$interval_files = $range_time[0].':'.$range_time[1];
}
} else {
$interval_files = '.';
}
$command .= ' -R '.$interval_files.' -M '.$config['general_network_path'].$config['sflow_name_dir'];
}
}
@ -1045,7 +1083,19 @@ function netflow_get_command($options, $filter)
if (isset($config['netflow_name_dir']) && $config['netflow_name_dir'] !== ''
&& isset($config['general_network_path']) && $config['general_network_path'] !== ''
) {
$command .= ' -R. -M '.$config['general_network_path'].$config['netflow_name_dir'];
if ($date_init > 0 && $date_end > 0) {
$path = $config['general_network_path'].$config['netflow_name_dir'];
$range_time = find_range_files_time($path, $date_init, $date_end);
if ($range_time[0] === 0 && $range_time[1] === 0) {
$interval_files = '.';
} else {
$interval_files = $range_time[0].':'.$range_time[1];
}
} else {
$interval_files = '.';
}
$command .= ' -R '.$interval_files.' -M '.$config['general_network_path'].$config['netflow_name_dir'];
}
}
}
@ -1059,6 +1109,62 @@ function netflow_get_command($options, $filter)
}
/**
* Find the two files closest to the time range.
*
* @param string $folder Folder of netflow.
* @param integer $date_init Time init for range.
* @param integer $date_end Time end for range.
*
* @return array
*/
function find_range_files_time($folder, $date_init, $date_end)
{
$closest_init = 0;
$closest_end = 0;
$closest_init_date = 0;
$closest_end_date = 0;
$min_diff_init = PHP_INT_MAX;
$min_diff_end = PHP_INT_MAX;
$files = scandir($folder);
if ($date_init > 0) {
foreach ($files as $file) {
if (preg_match('/^nfcapd\.(\d{12})$/', $file, $matches)) {
$file_date = $matches[1];
$file_date = strtotime(substr($file_date, 0, 4).'-'.substr($file_date, 4, 2).'-'.substr($file_date, 6, 2).' '.substr($file_date, 8, 2).':'.substr($file_date, 10, 2));
$diff_init = abs($file_date - $date_init);
if ($diff_init < $min_diff_init) {
$closest_init_date = $file_date;
$min_diff_init = $diff_init;
$closest_init = $file;
}
$diff_end = abs($file_date - $date_end);
if ($diff_end < $min_diff_end) {
$closest_end_date = $file_date;
$min_diff_end = $diff_end;
$closest_end = $file;
}
}
}
}
if ($closest_end_date < $date_init || $closest_init_date > $date_end) {
return [
0,
0,
];
} else {
return [
$closest_init,
$closest_end,
];
}
}
/**
* Returns the nfdump command line arguments that match the given filter.
*
@ -1336,7 +1442,9 @@ function netflow_draw_item(
$max_aggregates,
true,
$connection_name,
$address_resolution
$address_resolution,
$start_date,
$end_date
);
if (empty($data_pie) === true) {
@ -1451,7 +1559,9 @@ function netflow_draw_item(
$max_aggregates,
true,
$connection_name,
$address_resolution
$address_resolution,
$start_date,
$end_date
);
if (empty($data_stats) === false) {
@ -1572,7 +1682,9 @@ function netflow_get_item_data(
$aggregate,
$max_aggregates,
true,
$connection_name
$connection_name,
$start_date,
$end_date
);
$data = [
@ -1801,7 +1913,7 @@ function netflow_get_top_summary(
}
$options = "-q -o csv -n $max -s $sort/$order_text -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command = netflow_get_command($options, $netflow_filter);
$command = netflow_get_command($options, $netflow_filter, $start_date, $end_date);
exec($command, $result);
if (! is_array($result)) {
@ -1962,7 +2074,7 @@ function netflow_get_top_data(
date($nfdump_date_format, $start_date),
date($nfdump_date_format, $end_date)
);
$agg_command = netflow_get_command($options, $filter);
$agg_command = netflow_get_command($options, $filter, $start_date, $end_date);
// Call nfdump.
exec($agg_command, $string);

77
pandora_console/operation/netflow/nf_live_view.php
View File

@ -109,41 +109,64 @@ $interval_length = get_parameter('interval_length', NETFLOW_RES_MEDD);
$address_resolution = (int) get_parameter('address_resolution', ($config['netflow_get_ip_hostname'] ?? ''));
$filter_selected = (int) get_parameter('filter_selected', 0);
// Calculate range dates.
$custom_date = get_parameter('custom_date', '0');
$date = get_parameter('date', SECONDS_1DAY);
$date_end = get_parameter('date_end', 0);
$time_end = get_parameter('time_end');
$datetime_end = strtotime($date_end.' '.$time_end);
$custom_date = get_parameter('custom_date', 0);
$range = get_parameter('date', SECONDS_1DAY);
$date_text = get_parameter('date_text', SECONDS_1DAY);
$date_init_less = (strtotime(date('Y/m/d')) - SECONDS_1DAY);
$date_init = get_parameter('date_init', date(DATE_FORMAT, $date_init_less));
$time_init = get_parameter('time_init', date(TIME_FORMAT, $date_init_less));
$datetime_init = strtotime($date_init.' '.$time_init);
if ($custom_date === '1') {
$date_init = get_parameter('date_init');
$time_init = get_parameter('time_init');
$date_end = get_parameter('date_end');
$time_end = get_parameter('time_end');
$date_from = strtotime($date_init.' '.$time_init);
$date_to = strtotime($date_end.' '.$time_end);
if ($datetime_init >= $datetime_end) {
$datetime_init = $date_init_less;
}
$date_init = date('Y/m/d H:i:s', $datetime_init);
$date_end = date('Y/m/d H:i:s', $datetime_end);
$period = ($datetime_end - $datetime_init);
} else if ($custom_date === '2') {
$date_text = get_parameter('date_text');
$date_units = get_parameter('date_units');
$period = ($date_text * $date_units);
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = (strtotime($date_to) - $period);
} else if (in_array($date, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($date === 'this_week') {
$date_from = strtotime('last monday');
$date_to = strtotime($date_from.' +6 days');
} else if ($date === 'this_month') {
$date_from = strtotime('first day of this month');
$date_to = strtotime('last day of this month');
} else if ($date === 'past_month') {
$date_from = strtotime('first day of previous month');
$date_to = strtotime('last day of previous month');
} else if ($date === 'past_week') {
$date_from = strtotime('monday', strtotime('last week'));
$date_to = strtotime('sunday', strtotime('last week'));
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - ((int) $date_text * (int) $date_units)));
$period = (strtotime($date_end) - strtotime($date_init));
} else if (in_array($range, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($range === 'this_week') {
$monday = date('Y/m/d', strtotime('last monday'));
$sunday = date('Y/m/d', strtotime($monday.' +6 days'));
$period = (strtotime($sunday) - strtotime($monday));
$date_init = $monday;
$date_end = $sunday;
} else if ($range === 'this_month') {
$date_end = date('Y/m/d', strtotime('last day of this month'));
$first_of_month = date('Y/m/d', strtotime('first day of this month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_month') {
$date_end = date('Y/m/d', strtotime('last day of previous month'));
$first_of_month = date('Y/m/d', strtotime('first day of previous month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_week') {
$date_end = date('Y/m/d', strtotime('sunday', strtotime('last week')));
$first_of_week = date('Y/m/d', strtotime('monday', strtotime('last week')));
$date_init = $first_of_week;
$period = (strtotime($date_end) - strtotime($first_of_week));
}
} else {
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = ($date_to - $date);
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - $range));
$period = (strtotime($date_end) - strtotime($date_init));
}
$date_from = strtotime($date_init);
$date_to = strtotime($date_end);
// Read buttons.
$draw = get_parameter('draw_button', '');
$save = get_parameter('save_button', '');

77
pandora_console/operation/network/network_report.php
View File

@ -59,40 +59,63 @@ ui_include_time_picker();
// Calculate range dates.
$custom_date = get_parameter('custom_date', '0');
$date = get_parameter('date', SECONDS_1DAY);
$date_end = get_parameter('date_end', 0);
$time_end = get_parameter('time_end');
$datetime_end = strtotime($date_end.' '.$time_end);
$custom_date = get_parameter('custom_date', 0);
$range = get_parameter('date', SECONDS_1DAY);
$date_text = get_parameter('date_text', SECONDS_1DAY);
$date_init_less = (strtotime(date('Y/m/d')) - SECONDS_1DAY);
$date_init = get_parameter('date_init', date(DATE_FORMAT, $date_init_less));
$time_init = get_parameter('time_init', date(TIME_FORMAT, $date_init_less));
$datetime_init = strtotime($date_init.' '.$time_init);
if ($custom_date === '1') {
$date_init = get_parameter('date_init');
$time_init = get_parameter('time_init');
$date_end = get_parameter('date_end');
$time_end = get_parameter('time_end');
$date_from = strtotime($date_init.' '.$time_init);
$date_to = strtotime($date_end.' '.$time_end);
if ($datetime_init >= $datetime_end) {
$datetime_init = $date_init_less;
}
$date_init = date('Y/m/d H:i:s', $datetime_init);
$date_end = date('Y/m/d H:i:s', $datetime_end);
$period = ($datetime_end - $datetime_init);
} else if ($custom_date === '2') {
$date_text = get_parameter('date_text');
$date_units = get_parameter('date_units');
$period = ($date_text * $date_units);
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = (strtotime($date_to) - $period);
} else if (in_array($date, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($date === 'this_week') {
$date_from = strtotime('last monday');
$date_to = strtotime($date_from.' +6 days');
} else if ($date === 'this_month') {
$date_from = strtotime('first day of this month');
$date_to = strtotime('last day of this month');
} else if ($date === 'past_month') {
$date_from = strtotime('first day of previous month');
$date_to = strtotime('last day of previous month');
} else if ($date === 'past_week') {
$date_from = strtotime('monday', strtotime('last week'));
$date_to = strtotime('sunday', strtotime('last week'));
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - ((int) $date_text * (int) $date_units)));
$period = (strtotime($date_end) - strtotime($date_init));
} else if (in_array($range, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($range === 'this_week') {
$monday = date('Y/m/d', strtotime('last monday'));
$sunday = date('Y/m/d', strtotime($monday.' +6 days'));
$period = (strtotime($sunday) - strtotime($monday));
$date_init = $monday;
$date_end = $sunday;
} else if ($range === 'this_month') {
$date_end = date('Y/m/d', strtotime('last day of this month'));
$first_of_month = date('Y/m/d', strtotime('first day of this month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_month') {
$date_end = date('Y/m/d', strtotime('last day of previous month'));
$first_of_month = date('Y/m/d', strtotime('first day of previous month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_week') {
$date_end = date('Y/m/d', strtotime('sunday', strtotime('last week')));
$first_of_week = date('Y/m/d', strtotime('monday', strtotime('last week')));
$date_init = $first_of_week;
$period = (strtotime($date_end) - strtotime($first_of_week));
}
} else {
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = ($date_to - $date);
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - $range));
$period = (strtotime($date_end) - strtotime($date_init));
}
$date_from = strtotime($date_init);
$date_to = strtotime($date_end);
$filter_id = (int) get_parameter('filter_id', 0);
// Query params and other initializations.

77
pandora_console/operation/network/network_usage_map.php
View File

@ -85,40 +85,63 @@ ui_include_time_picker();
// Query params and other initializations.
$action = get_parameter('action', 'talkers');
// Calculate range dates.
$custom_date = get_parameter('custom_date', '0');
$date = get_parameter('date', SECONDS_1DAY);
$date_end = get_parameter('date_end', 0);
$time_end = get_parameter('time_end');
$datetime_end = strtotime($date_end.' '.$time_end);
$custom_date = get_parameter('custom_date', 0);
$range = get_parameter('date', SECONDS_1DAY);
$date_text = get_parameter('date_text', SECONDS_1DAY);
$date_init_less = (strtotime(date('Y/m/d')) - SECONDS_1DAY);
$date_init = get_parameter('date_init', date(DATE_FORMAT, $date_init_less));
$time_init = get_parameter('time_init', date(TIME_FORMAT, $date_init_less));
$datetime_init = strtotime($date_init.' '.$time_init);
if ($custom_date === '1') {
$date_init = get_parameter('date_init');
$time_init = get_parameter('time_init');
$date_end = get_parameter('date_end');
$time_end = get_parameter('time_end');
$date_from = strtotime($date_init.' '.$time_init);
$date_to = strtotime($date_end.' '.$time_end);
if ($datetime_init >= $datetime_end) {
$datetime_init = $date_init_less;
}
$date_init = date('Y/m/d H:i:s', $datetime_init);
$date_end = date('Y/m/d H:i:s', $datetime_end);
$period = ($datetime_end - $datetime_init);
} else if ($custom_date === '2') {
$date_text = get_parameter('date_text');
$date_units = get_parameter('date_units');
$period = ($date_text * $date_units);
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = (strtotime($date_to) - $period);
} else if (in_array($date, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($date === 'this_week') {
$date_from = strtotime('last monday');
$date_to = strtotime($date_from.' +6 days');
} else if ($date === 'this_month') {
$date_from = strtotime('first day of this month');
$date_to = strtotime('last day of this month');
} else if ($date === 'past_month') {
$date_from = strtotime('first day of previous month');
$date_to = strtotime('last day of previous month');
} else if ($date === 'past_week') {
$date_from = strtotime('monday', strtotime('last week'));
$date_to = strtotime('sunday', strtotime('last week'));
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - ((int) $date_text * (int) $date_units)));
$period = (strtotime($date_end) - strtotime($date_init));
} else if (in_array($range, ['this_week', 'this_month', 'past_week', 'past_month'])) {
if ($range === 'this_week') {
$monday = date('Y/m/d', strtotime('last monday'));
$sunday = date('Y/m/d', strtotime($monday.' +6 days'));
$period = (strtotime($sunday) - strtotime($monday));
$date_init = $monday;
$date_end = $sunday;
} else if ($range === 'this_month') {
$date_end = date('Y/m/d', strtotime('last day of this month'));
$first_of_month = date('Y/m/d', strtotime('first day of this month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_month') {
$date_end = date('Y/m/d', strtotime('last day of previous month'));
$first_of_month = date('Y/m/d', strtotime('first day of previous month'));
$date_init = $first_of_month;
$period = (strtotime($date_end) - strtotime($first_of_month));
} else if ($range === 'past_week') {
$date_end = date('Y/m/d', strtotime('sunday', strtotime('last week')));
$first_of_week = date('Y/m/d', strtotime('monday', strtotime('last week')));
$date_init = $first_of_week;
$period = (strtotime($date_end) - strtotime($first_of_week));
}
} else {
$date_to = strtotime(date('Y-m-d H:i:s'));
$date_from = ($date_to - $date);
$date_end = date('Y/m/d H:i:s');
$date_init = date('Y/m/d H:i:s', (strtotime($date_end) - $range));
$period = (strtotime($date_end) - strtotime($date_init));
}
$date_from = strtotime($date_init);
$date_to = strtotime($date_end);
$advanced_filter = get_parameter('advanced_filter', '');
$top = (int) get_parameter('top', 10);