From 57d91d50c32b46772df050ee99f5edc20d24974c Mon Sep 17 00:00:00 2001 From: Ramon Novoa Date: Fri, 13 Jan 2012 13:14:16 +0000 Subject: [PATCH] 2012-01-13 Ramon Novoa * include/functions_graph.php, include/functions_netflow.php, operation/netflow/nf_view.php: Fixed data source ordering for some netflow charts and implemented a basic disk cache. git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@5368 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f --- pandora_console/ChangeLog | 8 + pandora_console/include/functions_graph.php | 1 + pandora_console/include/functions_netflow.php | 400 ++++++++++++------ pandora_console/operation/netflow/nf_view.php | 10 +- 4 files changed, 295 insertions(+), 124 deletions(-) diff --git a/pandora_console/ChangeLog b/pandora_console/ChangeLog index b5688cdab0..0735ab376c 100644 --- a/pandora_console/ChangeLog +++ b/pandora_console/ChangeLog @@ -1,3 +1,10 @@ +2012-01-13 Ramon Novoa + + * include/functions_graph.php, + include/functions_netflow.php, + operation/netflow/nf_view.php: Fixed data source ordering for some + netflow charts and implemented a basic disk cache. + 2012-01-13 Vanessa Gil * godmode/netflow/nf_item_list.php: Changes sort @@ -30,6 +37,7 @@ set_delete_module_template, set_update_user, set_add_user_profile, set_create_module_template. +>>>>>>> .r5367 2012-01-12 Vanessa Gil * godmode/netflow/nf_edit.php diff --git a/pandora_console/include/functions_graph.php b/pandora_console/include/functions_graph.php index ebb946a61f..c2e23681cd 100755 --- a/pandora_console/include/functions_graph.php +++ b/pandora_console/include/functions_graph.php @@ -1966,6 +1966,7 @@ function graph_netflow_aggregate_area ($data, $period, $width, $height, $title, // Calculate source indexes $i = 0; + $sources = array (); foreach ($data['sources'] as $source => $value) { $source_indexes[$source] = $i; $sources[$i] = $source; diff --git a/pandora_console/include/functions_netflow.php b/pandora_console/include/functions_netflow.php index 12222db260..90a29b0b36 100644 --- a/pandora_console/include/functions_netflow.php +++ b/pandora_console/include/functions_netflow.php @@ -171,7 +171,7 @@ function netflow_reports_get_content ($id_rc, $filter = false, $fields = false){ * @return Result of the comparison. */ function compare_flows ($a, $b) { - return $a['data']>$b['data']; + return $a['data'] < $b['data']; } /** @@ -180,7 +180,7 @@ function compare_flows ($a, $b) { * @param array netflow_data Netflow data array. * */ -function sort_netflow_data ($netflow_data) { +function sort_netflow_data (&$netflow_data) { usort($netflow_data, "compare_flows"); } @@ -308,6 +308,7 @@ function netflow_is_net ($address) { * @param string start_date Period start date. * @param string end_date Period end date. * @param string command Command used to retrieve netflow data. + * @param string unique_id A unique number that is used to generate a cache file. * @param string aggregate Aggregate field. * @param int max Maximum number of aggregates. * @param string unit Unit to show. @@ -315,14 +316,15 @@ function netflow_is_net ($address) { * @return An array with netflow stats. * */ -function netflow_get_data ($start_date, $end_date, $command, $aggregate, $max, $unit){ +function netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max, $unit) { global $nfdump_date_format; global $config; // If there is aggregation calculate the top n - $sources = array (); if ($aggregate != 'none') { - $agg_command = $command . " -s $aggregate -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); + $values['data'] = array (); + $values['sources'] = array (); + $agg_command = $command . " -s $aggregate/$unit -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); exec($agg_command, $string); foreach($string as $line){ if ($line=='') { @@ -331,131 +333,35 @@ function netflow_get_data ($start_date, $end_date, $command, $aggregate, $max, $ $line = preg_replace('/\(\s*\S+\)/','',$line); $line = preg_replace('/\s+/',' ',$line); $val = explode(' ',$line); - $sources[$val[4]] = 1; + $values['sources'][$val[4]] = 1; } + } else { + $values = array (); } - // Execute nfdump and save its output in a temporary file - $command .= ' -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date); - $temp_file = $config["attachment_store"] . "/netflow_" . rand (0, getrandmax()) . ".data"; - exec("$command > $temp_file"); + // Load cache + $cache_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.cache'; + $last_timestamp = netflow_load_cache ($values, $cache_file, $start_date, $end_date, $aggregate); + if ($last_timestamp < $end_date) { - // Open the temporary file - $fh = fopen ($temp_file, "r"); - if ($fh === FALSE) { - return; - } + $last_timestamp++; - // Calculate the number of intervals - $num_intervals = $config['graph_res'] * 50; - $period = $end_date - $start_date; - $interval_length = (int) ($period / $num_intervals); + // Execute nfdump and save its output in a temporary file + $temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp'; + $command .= ' -t '.date($nfdump_date_format, $last_timestamp).'-'.date($nfdump_date_format, $end_date); + exec("$command > $temp_file"); - // Parse flow data - $read_flag = 1; - $values = array (); - $flow = array (); - for ($i = 0; $i < $num_intervals; $i++) { - $timestamp = $start_date + ($interval_length * $i); + // Parse data file + // We must parse from $start_date to avoid creating new intervals! + netflow_parse_file ($start_date, $end_date, $temp_file, $values, $aggregate, $unit); - if ($aggregate != 'none') { - $interval_total = array (); - $interval_count = array (); - } else { - $interval_total = 0; - $interval_count = 0; - } - - do { - if ($read_flag == 1) { - $read_flag = 0; - $line = fgets($fh, 4096); - if ($line === false) { - $read_flag = 1; - break; - } - - $line = preg_replace('/\s+/',' ',$line); - $val = explode(' ',$line); - if (! isset ($val[6])) { - $read_flag = 1; - break; - } - - $flow['date'] = $val[0]; - $flow['time'] = $val[1]; - - switch ($aggregate){ - case "proto": - $flow['agg'] = $val[3]; - break; - case "srcip": - $val2 = explode(':', $val[4]); - $flow['agg'] = $val2[0]; - break; - case "srcport": - $val2 = explode(':', $val[4]); - $flow['agg'] = $val2[1]; - break; - case "dstip": - $val2 = explode(':', $val[6]); - $flow['agg'] = $val2[0]; - break; - case "dstport": - $val2 = explode(':', $val[6]); - $flow['agg'] = $val2[1]; - break; - } - - switch ($unit) { - case "packets": - $flow['data'] = $val[7]; - break; - case "bytes": - $flow['data'] = $val[8]; - break; - case "flows": - $flow['data'] = $val[9]; - break; - } - $flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']); - } - if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) { - $read_flag = 1; - if ($aggregate != 'none') { - if (isset ($sources[$flow['agg']])) { - if (! isset ($interval_total[$flow['agg']])) { - $interval_total[$flow['agg']] = 0; - $interval_count[$flow['agg']] = 0; - } - $interval_total[$flow['agg']] += $flow['data']; - $interval_count[$flow['agg']] += 1; - } - } else { - $interval_total += $flow['data']; - $interval_count += 1; - } - } - } while ($read_flag == 1); - - if ($aggregate != 'none') { - foreach ($interval_total as $agg => $val) { - if ($interval_count[$agg] != 0) { - $values['data'][$timestamp][$agg] = (int) ($interval_total[$agg] / $interval_count[$agg]); - $values['sources'][$agg] = 1; - } - } - } else { - if ($interval_count == 0) { - $values[$timestamp]['data'] = 0; - } else { - $values[$timestamp]['data'] = (int) ($interval_total / $interval_count); - } - } + unlink ($temp_file); } - fclose ($fh); - unlink ($temp_file); + // Save cache + if ($aggregate == 'none') { + netflow_save_cache ($values, $cache_file); + } return $values; } @@ -619,4 +525,256 @@ function netflow_get_command ($filter) { return $command; } + +/** + * Parses netflow data from the given file. + * + * @param string start_date Period start date. + * @param string end_date Period end date. + * @param string file File that contains netflow data. + * @param array values Array where netflow data will be placed. + * @param string aggregate Aggregate field. + * @param string unit Unit to show. + * + * @return Timestamp of the last data read. + * + */ +function netflow_parse_file ($start_date, $end_date, $file, &$values, $aggregate, $unit) { + global $config; + + // Last timestamp read + $last_timestamp = $start_date; + + // Open the data file + $fh = @fopen ($file, "r"); + if ($fh === FALSE) { + return $last_timestamp; + } + + // Calculate the number of intervals + $num_intervals = $config['graph_res'] * 50; + $period = $end_date - $start_date; + $interval_length = (int) ($period / $num_intervals); + + // Parse flow data + $read_flag = 1; + $flow = array (); + for ($i = 0; $i < $num_intervals; $i++) { + $timestamp = $start_date + ($interval_length * $i); + + if ($aggregate != 'none') { + $interval_total = array (); + $interval_count = array (); + } else { + $interval_total = 0; + $interval_count = 0; + } + + do { + if ($read_flag == 1) { + $read_flag = 0; + $line = fgets($fh, 4096); + if ($line === false) { + $read_flag = 1; + break; + } + + $line = preg_replace('/\s+/',' ',$line); + $val = explode(' ',$line); + if (! isset ($val[6])) { + $read_flag = 1; + break; + } + + $flow['date'] = $val[0]; + $flow['time'] = $val[1]; + + switch ($aggregate){ + case "proto": + $flow['agg'] = $val[3]; + break; + case "srcip": + $val2 = explode(':', $val[4]); + $flow['agg'] = $val2[0]; + break; + case "srcport": + $val2 = explode(':', $val[4]); + $flow['agg'] = $val2[1]; + break; + case "dstip": + $val2 = explode(':', $val[6]); + $flow['agg'] = $val2[0]; + break; + case "dstport": + $val2 = explode(':', $val[6]); + $flow['agg'] = $val2[1]; + break; + } + + switch ($unit) { + case "packets": + $flow['data'] = $val[7]; + break; + case "bytes": + $flow['data'] = $val[8]; + break; + case "flows": + $flow['data'] = $val[9]; + break; + } + $flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']); + $last_timestamp = $flow['timestamp']; + } + if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) { + $read_flag = 1; + if ($aggregate != 'none') { + if (isset ($values['sources'][$flow['agg']])) { + if (! isset ($interval_total[$flow['agg']])) { + $interval_total[$flow['agg']] = 0; + $interval_count[$flow['agg']] = 0; + } + $interval_total[$flow['agg']] += $flow['data']; + $interval_count[$flow['agg']] += 1; + } + } else { + $interval_total += $flow['data']; + $interval_count += 1; + } + } + } while ($read_flag == 1); + + if ($aggregate != 'none') { + foreach ($interval_total as $agg => $val) { + + // No data for this interval/aggregate + if ($interval_count[$agg] == 0) { + continue; + } + + // Read previous data for this interval + if (isset ($values['data'][$timestamp][$agg])) { + $previous_value = $values['data'][$timestamp][$agg]; + } else { + $previous_value = 0; + } + + // Calculate interval data + $values['data'][$timestamp][$agg] = (int) ($interval_total[$agg] / $interval_count[$agg]); + + // Average with previous data + if ($previous_value != 0) { + $values['data'][$timestamp][$agg] = (int) (($values['data'][$timestamp][$agg] + $previous_data) / 2); + } + } + } else { + + // No data for this interval + if ($interval_count == 0) { + continue; + } + + // Read previous data for this interval + if (isset ($values[$timestamp]['data'])) { + $previous_value = $values[$timestamp]['data']; + } else { + $previous_value = 0; + } + + // Calculate interval data + $values[$timestamp]['data'] = (int) ($interval_total / $interval_count); + + // Average with previous data + if ($previous_value != 0) { + $values[$timestamp]['data'] = (int) (($values[$timestamp]['data'] + $previous_value) / 2); + } + + + } + } + + fclose ($fh); + return $last_timestamp; +} + +/** + * Save data to the specified cache file. + * + * @param array data Data array. + * @param string cache_file Cache file name. + * + */ +function netflow_save_cache ($data, $cache_file) { + + @file_put_contents ($cache_file, serialize ($data)); + + return; +} + +/** + * Load data from the specified cache file. + * + * @param string data Array were cache data will be stored. + * @param string cache_file Cache file name. + * @param string start_date Period start date. + * @param string end_date Period end date. + * + * @return Timestamp of the last data read from cache. + * + */ +function netflow_load_cache (&$data, $cache_file, $start_date, $end_date, $aggregate) { + global $config; + + // Open cache file + $cache_data = @file_get_contents ($cache_file); + $cache_data = @unserialize ($cache_data); + + // Calculate the number of intervals + $num_intervals = $config['graph_res'] * 50; + $period = $end_date - $start_date; + $interval_length = (int) ($period / $num_intervals); + $last_timestamp = $start_date; + + // Initializa chart data + if ($aggregate == 'none') { + if ($cache_data === FALSE) { + $cache_data = array (); + } + for ($i = 0; $i < $num_intervals; $i++) { + $timestamp = $start_date + ($interval_length * $i); + $interval_count = 0; + $interval_total = 0; + foreach ($cache_data as $cache_timestamp => $cache_value) { + if ($cache_timestamp < $timestamp + $interval_length) { + if ($cache_timestamp >= $timestamp) { + $interval_count++; + $interval_total += $cache_value['data']; + $last_timestamp = $cache_timestamp; + } + unset ($cache_data[$cache_timestamp]); + } else { + break; + } + } + + if ($interval_count > 0) { + $data[$timestamp]['data'] = (int) ($interval_total / $interval_count); + } else { + $data[$timestamp]['data'] = 0; + } + } + } else { + for ($i = 0; $i < $num_intervals; $i++) { + $timestamp = $start_date + ($interval_length * $i); + $interval_count = array (); + $interval_total = array (); + + foreach ($data['sources'] as $source => $null) { + $data['data'][$timestamp][$source] = 0; + } + } + } + + return $last_timestamp; +} + ?> diff --git a/pandora_console/operation/netflow/nf_view.php b/pandora_console/operation/netflow/nf_view.php index f38e8bbb2f..4889cc5b61 100644 --- a/pandora_console/operation/netflow/nf_view.php +++ b/pandora_console/operation/netflow/nf_view.php @@ -127,7 +127,9 @@ if (empty ($all_rcs)) { for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) { // Get report item - $content_report = db_get_row_sql("SELECT * FROM tnetflow_report_content WHERE id_rc='" . $all_rcs[$x]['id_rc'] . "'"); + $report_id = $all_rcs[$x]['id_rc']; + $content_report = db_get_row_sql("SELECT * FROM tnetflow_report_content WHERE id_rc='$report_id'"); + $content_id = $content_report['id_rc']; $max_aggregates= $content_report['max']; $type = $content_report['show_graph']; @@ -140,7 +142,8 @@ for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) { // Process item switch ($type){ case '0': - $data = netflow_get_data ($start_date, $end_date, $command, $aggregate, $max_aggregates, $unit); + $unique_id = $report_id . '_' . $content_id . '_' . ($end_date - $start_date); + $data = netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max_aggregates, $unit); if ($aggregate != 'none') { echo graph_netflow_aggregate_area($data, $interval, 660, 320, '', '', '', '', $end_date, $unit); } else { @@ -152,7 +155,8 @@ for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) { echo graph_netflow_aggregate_pie($result); break; case '2': - $data = netflow_get_data ($start_date, $end_date, $command, $aggregate, $max_aggregates, $unit); + $unique_id = $report_id . '_' . $content_id . '_' . ($end_date - $start_date); + $data = netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max_aggregates, $unit); echo netflow_data_table ($data, $start_date, $end_date, $unit); break; case '3':