2012-12-05 Ramon Novoa <rnovoa@artica.es>

* pandoradb_data.sql,
	  include/functions_config.php,
	  include/functions_netflow.php,
	  operation/netflow/nf_live_view.php,
	  pandoradb.data.postgreSQL.sql,
	  pandoradb.data.oracle.sql,
	  godmode/setup/setup_netflow.php,
	  godmode/netflow/nf_edit_form.php: Performance improvements. Use csv
	  output to avoid using regular expressions. Removed byte, flow and
	  packet unit and added kilobyte, megabyte, kilobytes per second and
	  megabytes per second.



git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@7232 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f
This commit is contained in:
Ramon Novoa 2012-12-05 17:29:06 +00:00
parent a973eec629
commit 278f4ee169
9 changed files with 189 additions and 113 deletions

View File

@ -1,3 +1,17 @@
2012-12-05 Ramon Novoa <rnovoa@artica.es>
* pandoradb_data.sql,
include/functions_config.php,
include/functions_netflow.php,
operation/netflow/nf_live_view.php,
pandoradb.data.postgreSQL.sql,
pandoradb.data.oracle.sql,
godmode/setup/setup_netflow.php,
godmode/netflow/nf_edit_form.php: Performance improvements. Use csv
output to avoid using regular expressions. Removed byte, flow and
packet unit and added kilobyte, megabyte, kilobytes per second and
megabytes per second.
2012-12-05 Sergio Martin <sergio.martin@artica.es>
* godmode/agentes/module_manager_editor_common.php

View File

@ -209,14 +209,12 @@ $table->data[7][0] = ui_print_help_icon ('pcap_filter', true);
$table->data[7][1] = html_print_textarea ('advanced_filter', 4, 40, $advanced_filter, '', true);
$table->data[8][0] = '<b>'.__('Aggregate by').'</b>'. ui_print_help_icon ('aggregate_by', true);
$aggregate_list = array();
$aggregate_list = array ('none' => __('None'), 'proto' => __('Protocol'), 'srcip' =>__('Src Ip Address'), 'dstip' =>__('Dst Ip Address'), 'srcport' =>__('Src Port'), 'dstport' =>__('Dst Port') );
$table->data[8][1] = html_print_select ($aggregate_list, "aggregate", $aggregate, '', '', 0, true, false, true, '', false);
$table->data[9][0] = '<b>'.__('Output format').'</b>';
$show_output = array();
$show_output = array ('packets' => __('Packets'), 'bytes' => __('Bytes'), 'flows' =>__('Flows'));
$show_output = array ('kilobytes' => __('Kilobytes'), 'megabytes' => __('Megabytes'), 'kilobytespersecond' => __('Kilobytes per second'), 'megabytespersecond' => __('Megabytes per second'));
$table->data[9][1] = html_print_select ($show_output, 'output', $output, '', '', 0, true, false, true, '', false);
echo '<form method="post" action="' . $config['homeurl'] . 'index.php?sec=netf&sec2=' . $config['homedir'] . '/godmode/netflow/nf_edit_form&pure='.$pure.'">';
@ -274,4 +272,4 @@ echo '</form>';
else {
displayAdvancedFilter ();
}
</script>
</script>

View File

@ -50,6 +50,8 @@ $table->data[1][0] = '<b>'.__('Daemon interval').'</b>';
$table->data[1][1] = html_print_input_text ('netflow_interval', $config['netflow_interval'], false, 50, 200, true);
$table->data[2][0] = '<b>'.__('Daemon binary path').'</b>';
$table->data[2][1] = html_print_input_text ('netflow_daemon', $config['netflow_daemon'], false, 50, 200, true);
$table->data[3][0] = '<b>'.__('Nfdump binary path').'</b>';
$table->data[3][1] = html_print_input_text ('netflow_nfdump', $config['netflow_nfdump'], false, 50, 200, true);
echo '<form id="netflow_setup" method="post">';

View File

@ -255,6 +255,7 @@ function config_update_config () {
config_update_value ('netflow_path', get_parameter ('netflow_path'));
config_update_value ('netflow_interval', get_parameter ('netflow_interval'));
config_update_value ('netflow_daemon', get_parameter ('netflow_daemon'));
config_update_value ('netflow_nfdump', get_parameter ('netflow_nfdump'));
break;
}
@ -498,6 +499,10 @@ function config_process_config () {
if (!isset ($config['netflow_daemon'])) {
config_update_value ( 'netflow_daemon', '/usr/bin/nfcapd');
}
if (!isset ($config['netflow_nfdump'])) {
config_update_value ( 'netflow_nfdump', '/usr/bin/nfdump');
}
if (!isset ($config['auth'])) {
config_update_value ( 'auth', 'mysql');

View File

@ -289,24 +289,39 @@ function netflow_data_table ($data, $start_date, $end_date, $aggregate) {
$source_index = array ();
$source_count = 0;
foreach ($data['sources'] as $source => $null) {
$table->align[$j+1] = "right";
$table->head[$j+1] = $source;
$source_index[$j] = $source;
$source_count++;
$j++;
}
$i = 0;
foreach ($data['data'] as $timestamp => $values) {
$table->data[$i][0] = date ($time_format, $timestamp);
for ($j = 0; $j < $source_count; $j++) {
if (isset ($values[$source_index[$j]])) {
$table->data[$i][$j+1] = format_numeric ($values[$source_index[$j]]);
}
else {
$table->data[$i][$j+1] = 0;
}
// No aggregates
if ($source_count == 0) {
$table->head[1] = __('Data');
$table->align[1] = "right";
$i = 0;
foreach ($data as $timestamp => $value) {
$table->data[$i][0] = date ($time_format, $timestamp);
$table->data[$i][1] = format_numeric ($value['data']);
$i++;
}
}
// Aggregates
else {
$i = 0;
foreach ($data['data'] as $timestamp => $values) {
$table->data[$i][0] = date ($time_format, $timestamp);
for ($j = 0; $j < $source_count; $j++) {
if (isset ($values[$source_index[$j]])) {
$table->data[$i][$j+1] = format_numeric ($values[$source_index[$j]]);
}
else {
$table->data[$i][$j+1] = 0;
}
}
$i++;
}
$i++;
}
return html_print_table($table, true);
@ -390,24 +405,30 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u
// Get the command to call nfdump
$command = netflow_get_command ($filter);
// Suppress the header line and the statistics at the bottom
$command .= ' -q';
// Suppress the header line and the statistics at the bottom and configure piped output
$command .= ' -q -o csv';
// If there is aggregation calculate the top n
if ($aggregate != 'none') {
$values['data'] = array ();
$values['sources'] = array ();
$agg_command = $command . " -s $aggregate/$unit -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$agg_command = $command . " -s $aggregate/bytes -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
exec($agg_command, $string);
// Reamove the first line
$string[0] = '';
foreach($string as $line){
if ($line=='') {
continue;
}
$line = preg_replace('/\(\s*\S+\)/','',$line);
$line = preg_replace('/\s+/',' ',$line);
$val = explode(' ',$line);
$values['sources'][$val[4]] = 1;
$val = explode(',',$line);
if ($aggregate == 'proto') {
$values['sources'][$val[3]] = 1;
} else {
$values['sources'][$val[4]] = 1;
}
}
}
else {
@ -425,7 +446,7 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u
$temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp';
$command .= ' -t '.date($nfdump_date_format, $last_timestamp).'-'.date($nfdump_date_format, $end_date);
exec("$command > $temp_file");
// Parse data file
// We must parse from $start_date to avoid creating new intervals!
netflow_parse_file ($start_date, $end_date, $interval_length, $temp_file, $values, $aggregate, $unit);
@ -437,7 +458,7 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u
if ($aggregate == 'none') {
netflow_save_cache ($values, $cache_file);
}
return $values;
}
@ -466,51 +487,62 @@ function netflow_get_stats ($start_date, $end_date, $filter, $aggregate, $max, $
$command = netflow_get_command ($filter);
// Execute nfdump
$command .= " -q -s $aggregate/$unit -n $max -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
$command .= " -o csv -q -s $aggregate/bytes -n $max -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
exec($command, $string);
if (! is_array($string)) {
return array ();
}
// Remove the first line
$string[0] = '';
$i = 0;
$values = array();
$interval_length = $end_date - $start_date;
foreach ($string as $line) {
if ($line == '') {
break;
continue;
}
$line = preg_replace('/\(\s*\S+\)/','',$line);
$line = preg_replace('/\s+/',' ',$line);
$val = explode(' ',$line);
$val = explode(',',$line);
$values[$i]['date'] = $val[0];
$values[$i]['time'] = $val[1];
//create field to sort array
$date = $val[0];
$time = $val[1];
$end_date = strtotime ($date." ".$time);
$datetime = $val[0];
$end_date = strtotime ($datetime);
$values[$i]['datetime'] = $end_date;
$values[$i]['agg'] = $val[4];
if (! isset ($val[7])) {
if ($aggregate == 'proto') {
$values[$i]['agg'] = $val[3];
} else {
$values[$i]['agg'] = $val[4];
}
if (! isset ($val[9])) {
return array ();
}
switch ($unit){
case "flows":
$values[$i]['data'] = $val[5];
case "megabytes":
$values[$i]['data'] = $val[9] / 1048576;
break;
case "packets":
$values[$i]['data'] = $val[6];
case "megabytespersecond":
$values[$i]['data'] = $val[9] / 1048576 / $interval_length;
break;
case "kilobytes":
$values[$i]['data'] = $val[9] / 1024;
break;
case "kilobytespersecond":
$values[$i]['data'] = $val[9] / 1024 / $interval_length;
break;
case "bytes":
default:
$values[$i]['data'] = $val[7];
$values[$i]['data'] = $val[9];
break;
}
$i++;
}
sort_netflow_data ($values);
return $values;
@ -543,7 +575,7 @@ function netflow_get_summary ($start_date, $end_date, $filter, $unique_id, $conn
$temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp';
$command .= " -o \"fmt: \" -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
exec("$command > $temp_file");
// Parse data file
// We must parse from $start_date to avoid creating new intervals!
$values = array ();
@ -565,8 +597,8 @@ function netflow_get_command ($filter) {
global $config;
// Build command
$command = 'nfdump -N -Otstart';
$command = $config['netflow_nfdump'] . ' -N -Otstart';
// Netflow data path
if (isset($config['netflow_path']) && $config['netflow_path'] != '') {
$command .= ' -R '.$config['netflow_path'];
@ -761,52 +793,37 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$
break;
}
$line = preg_replace('/\s+/',' ',$line);
$val = explode(' ',$line);
if (! isset ($val[6])) {
$val = explode(',',$line);;
if (! isset ($val[12])) {
$read_flag = 1;
break;
}
$flow['date'] = $val[0];
$flow['time'] = $val[1];
$flow['datetime'] = $val[0];
switch ($aggregate) {
case "proto":
$flow['agg'] = $val[3];
$flow['agg'] = $val[7];
break;
case "srcip":
$val2 = explode(':', $val[4]);
$flow['agg'] = $val2[0];
$flow['agg'] = $val[3];
break;
case "srcport":
$val2 = explode(':', $val[4]);
$flow['agg'] = $val2[1];
$flow['agg'] = $val[5];
break;
case "dstip":
$val2 = explode(':', $val[6]);
$flow['agg'] = $val2[0];
$flow['agg'] = $val[4];
break;
case "dstport":
$val2 = explode(':', $val[6]);
$flow['agg'] = $val2[1];
$flow['agg'] = $val[6];
break;
}
switch ($unit) {
case "flows":
$flow['data'] = $val[6];
break;
case "packets":
$flow['data'] = $val[7];
break;
case "bytes":
$flow['data'] = $val[8];
break;
}
$flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']);
$flow['data'] = $val[12];
$flow['timestamp'] = strtotime ($flow['datetime']);
$last_timestamp = $flow['timestamp'];
}
if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) {
$read_flag = 1;
if ($aggregate != 'none') {
@ -830,28 +847,32 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$
$no_data = 1;
if ($aggregate != 'none') {
foreach ($interval_total as $agg => $val) {
// No data for this interval/aggregate
if ($interval_count[$agg] == 0) {
continue;
}
// Read previous data for this interval
if (isset ($values['data'][$timestamp][$agg])) {
$previous_value = $values['data'][$timestamp][$agg];
}
else {
$previous_value = 0;
}
// Calculate interval data
$values['data'][$timestamp][$agg] = (int) $interval_total[$agg];
$no_data = 0;
// Add previous data
if ($previous_value != 0) {
$values['data'][$timestamp][$agg] = (int) ($values['data'][$timestamp][$agg] + $previous_data);
switch ($unit) {
case 'megabytes':
$values['data'][$timestamp][$agg] = $interval_total[$agg] / 1048576;
break;
case 'megabytespersecond':
$values['data'][$timestamp][$agg] = $interval_total[$agg] / 1048576 / $interval_length;
break;
case 'kilobytes':
$values['data'][$timestamp][$agg] = $interval_total[$agg] / 1024;
break;
case 'kilobytespersecond':
$values['data'][$timestamp][$agg] = $interval_total[$agg] / 1024 / $interval_length;
break;
default:
$values['data'][$timestamp][$agg] = $interval_total[$agg];
break;
}
$no_data = 0;
}
}
else {
@ -861,24 +882,26 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$
continue;
}
// Read previous data for this interval
if (isset ($values[$timestamp]['data'])) {
$previous_value = $values[$timestamp]['data'];
}
else {
$previous_value = 0;
}
// Calculate interval data
$values[$timestamp]['data'] = (int) $interval_total;
$no_data = 0;
// Add previous data
if ($previous_value != 0) {
$values[$timestamp]['data'] = (int) ($values[$timestamp]['data'] + $previous_value);
switch ($unit) {
case 'megabytes':
$values[$timestamp]['data'] = $interval_total / 1048576;
break;
case 'megabytespersecond':
$values[$timestamp]['data'] = $interval_total / 1048576 / $interval_length;
break;
case 'kilobytes':
$values[$timestamp]['data'] = $interval_total / 1024;
break;
case 'kilobytespersecond':
$values[$timestamp]['data'] = $interval_total / 1024 / $interval_length;
break;
default:
$values[$timestamp]['data'] = $interval_total;
break;
}
$no_data = 0;
}
}
@ -1028,6 +1051,34 @@ function netflow_get_valid_intervals () {
(string)SECONDS_2YEARS => __('2 years'));
}
/**
* Gets valid intervals for a netflow chart in the format:
*
* interval_length => interval_description
*
* @return Array of valid intervals.
*
*/
function netflow_get_valid_subintervals () {
return array (
(string)SECONDS_1MINUTE => __('1 min'),
(string)SECONDS_2MINUTES => __('2 mins'),
(string)SECONDS_5MINUTES => __('5 mins'),
(string)SECONDS_10MINUTES => __('10 mins'),
(string)SECONDS_15MINUTES => __('15 mins'),
(string)SECONDS_30MINUTES => __('30 mins'),
(string)SECONDS_1HOUR => __('1 hour'),
(string)SECONDS_2HOUR => __('2 hours'),
(string)SECONDS_5HOUR => __('5 hours'),
(string)SECONDS_12HOURS => __('12 hours'),
(string)SECONDS_1DAY => __('1 day'),
(string)SECONDS_2DAY => __('2 days'),
(string)SECONDS_5DAY => __('5 days'),
(string)SECONDS_15DAYS => __('15 days'),
(string)SECONDS_1WEEK => __('1 week'),
(string)SECONDS_1MONTH => __('1 month'));
}
/**
* Draw a netflow report item.
*

View File

@ -66,7 +66,7 @@ if (is_ajax()){
// Read filter configuration
$filter_id = (int) get_parameter ('filter_id', 0);
$filter['id_name'] = (string) get_parameter ('name', '');
$filter['id_name'] = get_parameter ('name', '');
$filter['id_group'] = (int) get_parameter ('assign_group', 0);
$filter['aggregate'] = get_parameter('aggregate','');
$filter['output'] = get_parameter('output','bytes');
@ -75,6 +75,7 @@ $filter['ip_src'] = get_parameter('ip_src','');
$filter['dst_port'] = get_parameter('dst_port','');
$filter['src_port'] = get_parameter('src_port','');
$filter['advanced_filter'] = get_parameter('advanced_filter','');
$filter['advanced_filter'] = get_parameter('advanced_filter','');
// Read chart configuration
$chart_type = (int) get_parameter('chart_type', 0);
@ -84,6 +85,7 @@ $update_date = (int) get_parameter('update_date', 0);
$date = get_parameter_post ('date', date ("Y/m/d", get_system_time ()));
$time = get_parameter_post ('time', date ("H:i:s", get_system_time ()));
$connection_name = get_parameter('connection_name', '');
$interval_length = (int) get_parameter('interval_length', 0);
// Read buttons
$draw = get_parameter('draw_button', '');
@ -157,8 +159,10 @@ echo '<form method="post" action="' . $config['homeurl'] . 'index.php?sec=netf&s
$table->data[0][2] = '<b>'.__('Interval').'</b>';
$table->data[0][3] = html_print_select (netflow_get_valid_intervals (), 'period', $period, '', '', 0, true, false, false);
$table->data[0][4] = '<b>'.__('Type').'</b>';
$table->data[0][5] = html_print_select (netflow_get_chart_types (), 'chart_type', $chart_type,'','',0,true);
$table->data[0][4] = '<b>'.__('Subinterval') . ui_print_help_tip (__("The interval will be divided in chunks the length of the subinterval"), true) . '</b>';
$table->data[0][5] = html_print_select (netflow_get_valid_subintervals (), 'interval_length', $interval_length, '', '', 0, true, false, false);
$table->data[0][6] = '<b>'.__('Type').'</b>';
$table->data[0][7] = html_print_select (netflow_get_chart_types (), 'chart_type', $chart_type,'','',0,true);
$max_values = array ('2' => '2',
'5' => '5',
'10' => '10',
@ -167,8 +171,8 @@ echo '<form method="post" action="' . $config['homeurl'] . 'index.php?sec=netf&s
'25' => '25',
'50' => '50'
);
$table->data[0][6] = '<b>'.__('Max. values').'</b>';
$table->data[0][7] = html_print_select ($max_values, 'max_aggregates', $max_aggregates, '', '', 0, true);
$table->data[0][8] = '<b>'.__('Max. values').'</b>';
$table->data[0][9] = html_print_select ($max_values, 'max_aggregates', $max_aggregates, '', '', 0, true);
if (defined ('METACONSOLE')) {
$table->data[0][8] = '<b>'.__('Connection').'</b>';
@ -232,8 +236,7 @@ echo '<form method="post" action="' . $config['homeurl'] . 'index.php?sec=netf&s
$table->data[6][1] = html_print_select ($aggregate_list, "aggregate", $filter['aggregate'], '', '', 0, true, false, true, '', false);
$table->data[6][2] = '<b>'.__('Output format').'</b>';
$show_output = array();
$show_output = array ('packets' => __('Packets'), 'bytes' => __('Bytes'), 'flows' =>__('Flows'));
$show_output = array ('kilobytes' => __('Kilobytes'), 'megabytes' => __('Megabytes'), 'kilobytespersecond' => __('Kilobytes per second'), 'megabytespersecond' => __('Megabytes per second'));
$table->data[6][3] = html_print_select ($show_output, 'output', $filter['output'], '', '', 0, true, false, true, '', false);
html_print_table ($table);
@ -254,7 +257,7 @@ if ($draw != '') {
$unique_id = 'live_view__' . ($end_date - $start_date);
// Draw
echo netflow_draw_item ($start_date, $end_date, 0, $chart_type, $filter, $max_aggregates, $unique_id, $connection_name);
echo netflow_draw_item ($start_date, $end_date, $interval_length, $chart_type, $filter, $max_aggregates, $unique_id, $connection_name);
}
?>

View File

@ -90,6 +90,7 @@ INSERT INTO tconfig (token, value) VALUES ('integria_url', ' ');
INSERT INTO tconfig (token, value) VALUES ('netflow_path', '/var/spool/pandora/data_in/netflow');
INSERT INTO tconfig (token, value) VALUES ('netflow_interval', '300');
INSERT INTO tconfig (token, value) VALUES ('netflow_daemon', '/usr/bin/nfcapd');
INSERT INTO tconfig (token, value) VALUES ('netflow_nfdump', '/usr/bin/nfdump');
INSERT INTO tconfig (token, value) VALUES ('event_fields', 'evento,id_agente,estado,timestamp');
INSERT INTO tconfig (token, value) VALUES ('list_ACL_IPs_for_API', '127.0.0.1');
INSERT INTO tconfig (token, value) VALUES ('enable_pass_policy', 0);

View File

@ -86,6 +86,7 @@ INSERT INTO "tconfig" ("token", "value") VALUES
('netflow_path', '/var/spool/pandora/data_in/netflow'),
('netflow_interval', '300'),
('netflow_daemon', '/usr/bin/nfcapd'),
('netflow_nfdump', '/usr/bin/nfdump'),
('event_fields', 'evento,id_agente,estado,timestamp'),
('list_ACL_IPs_for_API', '127.0.0.1'),
('enable_pass_policy', 0),

View File

@ -83,8 +83,9 @@ INSERT INTO `tconfig` (`token`, `value`) VALUES
('integria_inventory', '0'),
('integria_url', ''),
('netflow_path', '/var/spool/pandora/data_in/netflow'),
('netflow_interval', '300'),
('netflow_interval', '86400'),
('netflow_daemon', '/usr/bin/nfcapd'),
('netflow_nfdump', '/usr/bin/nfdump'),
('event_fields', 'evento,id_agente,estado,timestamp'),
('list_ACL_IPs_for_API', '127.0.0.1'),
('enable_pass_policy', 0),