From 278f4ee169c7c5d43ec7aaefdee4263f9ac4345b Mon Sep 17 00:00:00 2001 From: Ramon Novoa Date: Wed, 5 Dec 2012 17:29:06 +0000 Subject: [PATCH] 2012-12-05 Ramon Novoa * pandoradb_data.sql, include/functions_config.php, include/functions_netflow.php, operation/netflow/nf_live_view.php, pandoradb.data.postgreSQL.sql, pandoradb.data.oracle.sql, godmode/setup/setup_netflow.php, godmode/netflow/nf_edit_form.php: Performance improvements. Use csv output to avoid using regular expressions. Removed byte, flow and packet unit and added kilobyte, megabyte, kilobytes per second and megabytes per second. git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@7232 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f --- pandora_console/ChangeLog | 14 + .../godmode/netflow/nf_edit_form.php | 6 +- .../godmode/setup/setup_netflow.php | 2 + pandora_console/include/functions_config.php | 5 + pandora_console/include/functions_netflow.php | 251 +++++++++++------- .../operation/netflow/nf_live_view.php | 19 +- pandora_console/pandoradb.data.oracle.sql | 1 + pandora_console/pandoradb.data.postgreSQL.sql | 1 + pandora_console/pandoradb_data.sql | 3 +- 9 files changed, 189 insertions(+), 113 deletions(-) diff --git a/pandora_console/ChangeLog b/pandora_console/ChangeLog index 4bfa51c6f4..3b96db6277 100644 --- a/pandora_console/ChangeLog +++ b/pandora_console/ChangeLog @@ -1,3 +1,17 @@ +2012-12-05 Ramon Novoa + + * pandoradb_data.sql, + include/functions_config.php, + include/functions_netflow.php, + operation/netflow/nf_live_view.php, + pandoradb.data.postgreSQL.sql, + pandoradb.data.oracle.sql, + godmode/setup/setup_netflow.php, + godmode/netflow/nf_edit_form.php: Performance improvements. Use csv + output to avoid using regular expressions. Removed byte, flow and + packet unit and added kilobyte, megabyte, kilobytes per second and + megabytes per second. + 2012-12-05 Sergio Martin * godmode/agentes/module_manager_editor_common.php diff --git a/pandora_console/godmode/netflow/nf_edit_form.php b/pandora_console/godmode/netflow/nf_edit_form.php index 1b7ec49e74..b5032b9d64 100644 --- a/pandora_console/godmode/netflow/nf_edit_form.php +++ b/pandora_console/godmode/netflow/nf_edit_form.php @@ -209,14 +209,12 @@ $table->data[7][0] = ui_print_help_icon ('pcap_filter', true); $table->data[7][1] = html_print_textarea ('advanced_filter', 4, 40, $advanced_filter, '', true); $table->data[8][0] = ''.__('Aggregate by').''. ui_print_help_icon ('aggregate_by', true); -$aggregate_list = array(); $aggregate_list = array ('none' => __('None'), 'proto' => __('Protocol'), 'srcip' =>__('Src Ip Address'), 'dstip' =>__('Dst Ip Address'), 'srcport' =>__('Src Port'), 'dstport' =>__('Dst Port') ); $table->data[8][1] = html_print_select ($aggregate_list, "aggregate", $aggregate, '', '', 0, true, false, true, '', false); $table->data[9][0] = ''.__('Output format').''; -$show_output = array(); -$show_output = array ('packets' => __('Packets'), 'bytes' => __('Bytes'), 'flows' =>__('Flows')); +$show_output = array ('kilobytes' => __('Kilobytes'), 'megabytes' => __('Megabytes'), 'kilobytespersecond' => __('Kilobytes per second'), 'megabytespersecond' => __('Megabytes per second')); $table->data[9][1] = html_print_select ($show_output, 'output', $output, '', '', 0, true, false, true, '', false); echo '
'; @@ -274,4 +272,4 @@ echo '
'; else { displayAdvancedFilter (); } - \ No newline at end of file + diff --git a/pandora_console/godmode/setup/setup_netflow.php b/pandora_console/godmode/setup/setup_netflow.php index 6f93fb1c42..441ddfb6db 100644 --- a/pandora_console/godmode/setup/setup_netflow.php +++ b/pandora_console/godmode/setup/setup_netflow.php @@ -50,6 +50,8 @@ $table->data[1][0] = ''.__('Daemon interval').''; $table->data[1][1] = html_print_input_text ('netflow_interval', $config['netflow_interval'], false, 50, 200, true); $table->data[2][0] = ''.__('Daemon binary path').''; $table->data[2][1] = html_print_input_text ('netflow_daemon', $config['netflow_daemon'], false, 50, 200, true); +$table->data[3][0] = ''.__('Nfdump binary path').''; +$table->data[3][1] = html_print_input_text ('netflow_nfdump', $config['netflow_nfdump'], false, 50, 200, true); echo '
'; diff --git a/pandora_console/include/functions_config.php b/pandora_console/include/functions_config.php index c10eaeadcd..d4def54cdd 100644 --- a/pandora_console/include/functions_config.php +++ b/pandora_console/include/functions_config.php @@ -255,6 +255,7 @@ function config_update_config () { config_update_value ('netflow_path', get_parameter ('netflow_path')); config_update_value ('netflow_interval', get_parameter ('netflow_interval')); config_update_value ('netflow_daemon', get_parameter ('netflow_daemon')); + config_update_value ('netflow_nfdump', get_parameter ('netflow_nfdump')); break; } @@ -498,6 +499,10 @@ function config_process_config () { if (!isset ($config['netflow_daemon'])) { config_update_value ( 'netflow_daemon', '/usr/bin/nfcapd'); } + + if (!isset ($config['netflow_nfdump'])) { + config_update_value ( 'netflow_nfdump', '/usr/bin/nfdump'); + } if (!isset ($config['auth'])) { config_update_value ( 'auth', 'mysql'); diff --git a/pandora_console/include/functions_netflow.php b/pandora_console/include/functions_netflow.php index ba719d5462..94c930e431 100644 --- a/pandora_console/include/functions_netflow.php +++ b/pandora_console/include/functions_netflow.php @@ -289,24 +289,39 @@ function netflow_data_table ($data, $start_date, $end_date, $aggregate) { $source_index = array (); $source_count = 0; foreach ($data['sources'] as $source => $null) { + $table->align[$j+1] = "right"; $table->head[$j+1] = $source; $source_index[$j] = $source; $source_count++; $j++; } - - $i = 0; - foreach ($data['data'] as $timestamp => $values) { - $table->data[$i][0] = date ($time_format, $timestamp); - for ($j = 0; $j < $source_count; $j++) { - if (isset ($values[$source_index[$j]])) { - $table->data[$i][$j+1] = format_numeric ($values[$source_index[$j]]); - } - else { - $table->data[$i][$j+1] = 0; - } + + // No aggregates + if ($source_count == 0) { + $table->head[1] = __('Data'); + $table->align[1] = "right"; + $i = 0; + foreach ($data as $timestamp => $value) { + $table->data[$i][0] = date ($time_format, $timestamp); + $table->data[$i][1] = format_numeric ($value['data']); + $i++; + } + } + // Aggregates + else { + $i = 0; + foreach ($data['data'] as $timestamp => $values) { + $table->data[$i][0] = date ($time_format, $timestamp); + for ($j = 0; $j < $source_count; $j++) { + if (isset ($values[$source_index[$j]])) { + $table->data[$i][$j+1] = format_numeric ($values[$source_index[$j]]); + } + else { + $table->data[$i][$j+1] = 0; + } + } + $i++; } - $i++; } return html_print_table($table, true); @@ -390,24 +405,30 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u // Get the command to call nfdump $command = netflow_get_command ($filter); - // Suppress the header line and the statistics at the bottom - $command .= ' -q'; + // Suppress the header line and the statistics at the bottom and configure piped output + $command .= ' -q -o csv'; // If there is aggregation calculate the top n if ($aggregate != 'none') { $values['data'] = array (); $values['sources'] = array (); - $agg_command = $command . " -s $aggregate/$unit -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); + $agg_command = $command . " -s $aggregate/bytes -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); exec($agg_command, $string); + + // Reamove the first line + $string[0] = ''; foreach($string as $line){ if ($line=='') { continue; } - $line = preg_replace('/\(\s*\S+\)/','',$line); - $line = preg_replace('/\s+/',' ',$line); - $val = explode(' ',$line); - $values['sources'][$val[4]] = 1; + $val = explode(',',$line); + + if ($aggregate == 'proto') { + $values['sources'][$val[3]] = 1; + } else { + $values['sources'][$val[4]] = 1; + } } } else { @@ -425,7 +446,7 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u $temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp'; $command .= ' -t '.date($nfdump_date_format, $last_timestamp).'-'.date($nfdump_date_format, $end_date); exec("$command > $temp_file"); - + // Parse data file // We must parse from $start_date to avoid creating new intervals! netflow_parse_file ($start_date, $end_date, $interval_length, $temp_file, $values, $aggregate, $unit); @@ -437,7 +458,7 @@ function netflow_get_data ($start_date, $end_date, $interval_length, $filter, $u if ($aggregate == 'none') { netflow_save_cache ($values, $cache_file); } - + return $values; } @@ -466,51 +487,62 @@ function netflow_get_stats ($start_date, $end_date, $filter, $aggregate, $max, $ $command = netflow_get_command ($filter); // Execute nfdump - $command .= " -q -s $aggregate/$unit -n $max -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); + $command .= " -o csv -q -s $aggregate/bytes -n $max -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); exec($command, $string); - + if (! is_array($string)) { return array (); } - + + // Remove the first line + $string[0] = ''; + $i = 0; $values = array(); + $interval_length = $end_date - $start_date; foreach ($string as $line) { if ($line == '') { - break; + continue; } - $line = preg_replace('/\(\s*\S+\)/','',$line); - $line = preg_replace('/\s+/',' ',$line); - $val = explode(' ',$line); + + $val = explode(',',$line); $values[$i]['date'] = $val[0]; $values[$i]['time'] = $val[1]; //create field to sort array - $date = $val[0]; - $time = $val[1]; - $end_date = strtotime ($date." ".$time); + $datetime = $val[0]; + $end_date = strtotime ($datetime); $values[$i]['datetime'] = $end_date; - $values[$i]['agg'] = $val[4]; - if (! isset ($val[7])) { + if ($aggregate == 'proto') { + $values[$i]['agg'] = $val[3]; + } else { + $values[$i]['agg'] = $val[4]; + } + if (! isset ($val[9])) { return array (); } switch ($unit){ - case "flows": - $values[$i]['data'] = $val[5]; + case "megabytes": + $values[$i]['data'] = $val[9] / 1048576; break; - case "packets": - $values[$i]['data'] = $val[6]; + case "megabytespersecond": + $values[$i]['data'] = $val[9] / 1048576 / $interval_length; + break; + case "kilobytes": + $values[$i]['data'] = $val[9] / 1024; + break; + case "kilobytespersecond": + $values[$i]['data'] = $val[9] / 1024 / $interval_length; break; - case "bytes": default: - $values[$i]['data'] = $val[7]; + $values[$i]['data'] = $val[9]; break; } $i++; } - + sort_netflow_data ($values); return $values; @@ -543,7 +575,7 @@ function netflow_get_summary ($start_date, $end_date, $filter, $unique_id, $conn $temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp'; $command .= " -o \"fmt: \" -t " .date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); exec("$command > $temp_file"); - + // Parse data file // We must parse from $start_date to avoid creating new intervals! $values = array (); @@ -565,8 +597,8 @@ function netflow_get_command ($filter) { global $config; // Build command - $command = 'nfdump -N -Otstart'; - + $command = $config['netflow_nfdump'] . ' -N -Otstart'; + // Netflow data path if (isset($config['netflow_path']) && $config['netflow_path'] != '') { $command .= ' -R '.$config['netflow_path']; @@ -761,52 +793,37 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$ break; } - $line = preg_replace('/\s+/',' ',$line); - $val = explode(' ',$line); - if (! isset ($val[6])) { + $val = explode(',',$line);; + if (! isset ($val[12])) { $read_flag = 1; break; } - $flow['date'] = $val[0]; - $flow['time'] = $val[1]; + $flow['datetime'] = $val[0]; switch ($aggregate) { case "proto": - $flow['agg'] = $val[3]; + $flow['agg'] = $val[7]; break; case "srcip": - $val2 = explode(':', $val[4]); - $flow['agg'] = $val2[0]; + $flow['agg'] = $val[3]; break; case "srcport": - $val2 = explode(':', $val[4]); - $flow['agg'] = $val2[1]; + $flow['agg'] = $val[5]; break; case "dstip": - $val2 = explode(':', $val[6]); - $flow['agg'] = $val2[0]; + $flow['agg'] = $val[4]; break; case "dstport": - $val2 = explode(':', $val[6]); - $flow['agg'] = $val2[1]; + $flow['agg'] = $val[6]; break; } - switch ($unit) { - case "flows": - $flow['data'] = $val[6]; - break; - case "packets": - $flow['data'] = $val[7]; - break; - case "bytes": - $flow['data'] = $val[8]; - break; - } - $flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']); + $flow['data'] = $val[12]; + $flow['timestamp'] = strtotime ($flow['datetime']); $last_timestamp = $flow['timestamp']; } + if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) { $read_flag = 1; if ($aggregate != 'none') { @@ -830,28 +847,32 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$ $no_data = 1; if ($aggregate != 'none') { foreach ($interval_total as $agg => $val) { - + // No data for this interval/aggregate if ($interval_count[$agg] == 0) { continue; } - // Read previous data for this interval - if (isset ($values['data'][$timestamp][$agg])) { - $previous_value = $values['data'][$timestamp][$agg]; - } - else { - $previous_value = 0; - } - // Calculate interval data - $values['data'][$timestamp][$agg] = (int) $interval_total[$agg]; - $no_data = 0; - - // Add previous data - if ($previous_value != 0) { - $values['data'][$timestamp][$agg] = (int) ($values['data'][$timestamp][$agg] + $previous_data); + switch ($unit) { + case 'megabytes': + $values['data'][$timestamp][$agg] = $interval_total[$agg] / 1048576; + break; + case 'megabytespersecond': + $values['data'][$timestamp][$agg] = $interval_total[$agg] / 1048576 / $interval_length; + break; + case 'kilobytes': + $values['data'][$timestamp][$agg] = $interval_total[$agg] / 1024; + break; + case 'kilobytespersecond': + $values['data'][$timestamp][$agg] = $interval_total[$agg] / 1024 / $interval_length; + break; + default: + $values['data'][$timestamp][$agg] = $interval_total[$agg]; + break; } + + $no_data = 0; } } else { @@ -861,24 +882,26 @@ function netflow_parse_file ($start_date, $end_date, $interval_length, $file, &$ continue; } - // Read previous data for this interval - if (isset ($values[$timestamp]['data'])) { - $previous_value = $values[$timestamp]['data']; - } - else { - $previous_value = 0; - } - // Calculate interval data - $values[$timestamp]['data'] = (int) $interval_total; - $no_data = 0; - - // Add previous data - if ($previous_value != 0) { - $values[$timestamp]['data'] = (int) ($values[$timestamp]['data'] + $previous_value); + switch ($unit) { + case 'megabytes': + $values[$timestamp]['data'] = $interval_total / 1048576; + break; + case 'megabytespersecond': + $values[$timestamp]['data'] = $interval_total / 1048576 / $interval_length; + break; + case 'kilobytes': + $values[$timestamp]['data'] = $interval_total / 1024; + break; + case 'kilobytespersecond': + $values[$timestamp]['data'] = $interval_total / 1024 / $interval_length; + break; + default: + $values[$timestamp]['data'] = $interval_total; + break; } - - + + $no_data = 0; } } @@ -1028,6 +1051,34 @@ function netflow_get_valid_intervals () { (string)SECONDS_2YEARS => __('2 years')); } +/** + * Gets valid intervals for a netflow chart in the format: + * + * interval_length => interval_description + * + * @return Array of valid intervals. + * + */ +function netflow_get_valid_subintervals () { + return array ( + (string)SECONDS_1MINUTE => __('1 min'), + (string)SECONDS_2MINUTES => __('2 mins'), + (string)SECONDS_5MINUTES => __('5 mins'), + (string)SECONDS_10MINUTES => __('10 mins'), + (string)SECONDS_15MINUTES => __('15 mins'), + (string)SECONDS_30MINUTES => __('30 mins'), + (string)SECONDS_1HOUR => __('1 hour'), + (string)SECONDS_2HOUR => __('2 hours'), + (string)SECONDS_5HOUR => __('5 hours'), + (string)SECONDS_12HOURS => __('12 hours'), + (string)SECONDS_1DAY => __('1 day'), + (string)SECONDS_2DAY => __('2 days'), + (string)SECONDS_5DAY => __('5 days'), + (string)SECONDS_15DAYS => __('15 days'), + (string)SECONDS_1WEEK => __('1 week'), + (string)SECONDS_1MONTH => __('1 month')); +} + /** * Draw a netflow report item. * diff --git a/pandora_console/operation/netflow/nf_live_view.php b/pandora_console/operation/netflow/nf_live_view.php index 644398e936..a5b0c23ee6 100644 --- a/pandora_console/operation/netflow/nf_live_view.php +++ b/pandora_console/operation/netflow/nf_live_view.php @@ -66,7 +66,7 @@ if (is_ajax()){ // Read filter configuration $filter_id = (int) get_parameter ('filter_id', 0); -$filter['id_name'] = (string) get_parameter ('name', ''); +$filter['id_name'] = get_parameter ('name', ''); $filter['id_group'] = (int) get_parameter ('assign_group', 0); $filter['aggregate'] = get_parameter('aggregate',''); $filter['output'] = get_parameter('output','bytes'); @@ -75,6 +75,7 @@ $filter['ip_src'] = get_parameter('ip_src',''); $filter['dst_port'] = get_parameter('dst_port',''); $filter['src_port'] = get_parameter('src_port',''); $filter['advanced_filter'] = get_parameter('advanced_filter',''); +$filter['advanced_filter'] = get_parameter('advanced_filter',''); // Read chart configuration $chart_type = (int) get_parameter('chart_type', 0); @@ -84,6 +85,7 @@ $update_date = (int) get_parameter('update_date', 0); $date = get_parameter_post ('date', date ("Y/m/d", get_system_time ())); $time = get_parameter_post ('time', date ("H:i:s", get_system_time ())); $connection_name = get_parameter('connection_name', ''); +$interval_length = (int) get_parameter('interval_length', 0); // Read buttons $draw = get_parameter('draw_button', ''); @@ -157,8 +159,10 @@ echo ''; + $table->data[0][5] = html_print_select (netflow_get_valid_subintervals (), 'interval_length', $interval_length, '', '', 0, true, false, false); + $table->data[0][6] = ''.__('Type').''; + $table->data[0][7] = html_print_select (netflow_get_chart_types (), 'chart_type', $chart_type,'','',0,true); $max_values = array ('2' => '2', '5' => '5', '10' => '10', @@ -167,8 +171,8 @@ echo 'data[6][2] = ''.__('Output format').''; - $show_output = array(); - $show_output = array ('packets' => __('Packets'), 'bytes' => __('Bytes'), 'flows' =>__('Flows')); + $show_output = array ('kilobytes' => __('Kilobytes'), 'megabytes' => __('Megabytes'), 'kilobytespersecond' => __('Kilobytes per second'), 'megabytespersecond' => __('Megabytes per second')); $table->data[6][3] = html_print_select ($show_output, 'output', $filter['output'], '', '', 0, true, false, true, '', false); html_print_table ($table); @@ -254,7 +257,7 @@ if ($draw != '') { $unique_id = 'live_view__' . ($end_date - $start_date); // Draw - echo netflow_draw_item ($start_date, $end_date, 0, $chart_type, $filter, $max_aggregates, $unique_id, $connection_name); + echo netflow_draw_item ($start_date, $end_date, $interval_length, $chart_type, $filter, $max_aggregates, $unique_id, $connection_name); } ?> diff --git a/pandora_console/pandoradb.data.oracle.sql b/pandora_console/pandoradb.data.oracle.sql index 87b63be9ed..0c1f8cd667 100644 --- a/pandora_console/pandoradb.data.oracle.sql +++ b/pandora_console/pandoradb.data.oracle.sql @@ -90,6 +90,7 @@ INSERT INTO tconfig (token, value) VALUES ('integria_url', ' '); INSERT INTO tconfig (token, value) VALUES ('netflow_path', '/var/spool/pandora/data_in/netflow'); INSERT INTO tconfig (token, value) VALUES ('netflow_interval', '300'); INSERT INTO tconfig (token, value) VALUES ('netflow_daemon', '/usr/bin/nfcapd'); +INSERT INTO tconfig (token, value) VALUES ('netflow_nfdump', '/usr/bin/nfdump'); INSERT INTO tconfig (token, value) VALUES ('event_fields', 'evento,id_agente,estado,timestamp'); INSERT INTO tconfig (token, value) VALUES ('list_ACL_IPs_for_API', '127.0.0.1'); INSERT INTO tconfig (token, value) VALUES ('enable_pass_policy', 0); diff --git a/pandora_console/pandoradb.data.postgreSQL.sql b/pandora_console/pandoradb.data.postgreSQL.sql index 195555a2e6..592d6e490d 100644 --- a/pandora_console/pandoradb.data.postgreSQL.sql +++ b/pandora_console/pandoradb.data.postgreSQL.sql @@ -86,6 +86,7 @@ INSERT INTO "tconfig" ("token", "value") VALUES ('netflow_path', '/var/spool/pandora/data_in/netflow'), ('netflow_interval', '300'), ('netflow_daemon', '/usr/bin/nfcapd'), +('netflow_nfdump', '/usr/bin/nfdump'), ('event_fields', 'evento,id_agente,estado,timestamp'), ('list_ACL_IPs_for_API', '127.0.0.1'), ('enable_pass_policy', 0), diff --git a/pandora_console/pandoradb_data.sql b/pandora_console/pandoradb_data.sql index 46978d749e..771173fee1 100644 --- a/pandora_console/pandoradb_data.sql +++ b/pandora_console/pandoradb_data.sql @@ -83,8 +83,9 @@ INSERT INTO `tconfig` (`token`, `value`) VALUES ('integria_inventory', '0'), ('integria_url', ''), ('netflow_path', '/var/spool/pandora/data_in/netflow'), -('netflow_interval', '300'), +('netflow_interval', '86400'), ('netflow_daemon', '/usr/bin/nfcapd'), +('netflow_nfdump', '/usr/bin/nfdump'), ('event_fields', 'evento,id_agente,estado,timestamp'), ('list_ACL_IPs_for_API', '127.0.0.1'), ('enable_pass_policy', 0),