From 57d91d50c32b46772df050ee99f5edc20d24974c Mon Sep 17 00:00:00 2001
From: Ramon Novoa <rnovoa@artica.es>
Date: Fri, 13 Jan 2012 13:14:16 +0000
Subject: [PATCH] 2012-01-13  Ramon Novoa  <rnovoa@artica.es>

	* include/functions_graph.php,
	  include/functions_netflow.php,
	  operation/netflow/nf_view.php: Fixed data source ordering for some
	  netflow charts and implemented a basic disk cache.



git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@5368 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f
---
 pandora_console/ChangeLog                     |   8 +
 pandora_console/include/functions_graph.php   |   1 +
 pandora_console/include/functions_netflow.php | 400 ++++++++++++------
 pandora_console/operation/netflow/nf_view.php |  10 +-
 4 files changed, 295 insertions(+), 124 deletions(-)

diff --git a/pandora_console/ChangeLog b/pandora_console/ChangeLog
index b5688cdab0..0735ab376c 100644
--- a/pandora_console/ChangeLog
+++ b/pandora_console/ChangeLog
@@ -1,3 +1,10 @@
+2012-01-13  Ramon Novoa  <rnovoa@artica.es>
+
+	* include/functions_graph.php,
+	  include/functions_netflow.php,
+	  operation/netflow/nf_view.php: Fixed data source ordering for some
+	  netflow charts and implemented a basic disk cache.
+
 2012-01-13  Vanessa Gil  <vanessa.gil@artica.es>
 
 	* godmode/netflow/nf_item_list.php: Changes sort
@@ -30,6 +37,7 @@
 	 set_delete_module_template, set_update_user, set_add_user_profile,
 	 set_create_module_template.
 
+>>>>>>> .r5367
 2012-01-12  Vanessa Gil  <vanessa.gil@artica.es>
 
 	* godmode/netflow/nf_edit.php
diff --git a/pandora_console/include/functions_graph.php b/pandora_console/include/functions_graph.php
index ebb946a61f..c2e23681cd 100755
--- a/pandora_console/include/functions_graph.php
+++ b/pandora_console/include/functions_graph.php
@@ -1966,6 +1966,7 @@ function graph_netflow_aggregate_area ($data, $period, $width, $height, $title,
 
 	// Calculate source indexes
 	$i = 0;
+	$sources = array ();
 	foreach ($data['sources'] as $source => $value) {
 		$source_indexes[$source] = $i;
 		$sources[$i] = $source;
diff --git a/pandora_console/include/functions_netflow.php b/pandora_console/include/functions_netflow.php
index 12222db260..90a29b0b36 100644
--- a/pandora_console/include/functions_netflow.php
+++ b/pandora_console/include/functions_netflow.php
@@ -171,7 +171,7 @@ function netflow_reports_get_content ($id_rc, $filter = false, $fields = false){
  * @return Result of the comparison.
  */
 function compare_flows ($a, $b) {
-	return $a['data']>$b['data'];
+	return $a['data'] < $b['data'];
 }
 
 /**
@@ -180,7 +180,7 @@ function compare_flows ($a, $b) {
  * @param array netflow_data Netflow data array.
  *
  */
-function sort_netflow_data ($netflow_data) {
+function sort_netflow_data (&$netflow_data) {
      usort($netflow_data, "compare_flows");
 }
 
@@ -308,6 +308,7 @@ function netflow_is_net ($address) {
  * @param string start_date Period start date.
  * @param string end_date Period end date.
  * @param string command Command used to retrieve netflow data.
+ * @param string unique_id A unique number that is used to generate a cache file.
  * @param string aggregate Aggregate field.
  * @param int max Maximum number of aggregates.
  * @param string unit Unit to show.
@@ -315,14 +316,15 @@ function netflow_is_net ($address) {
  * @return An array with netflow stats.
  *
  */
-function netflow_get_data ($start_date, $end_date, $command, $aggregate, $max, $unit){
+function netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max, $unit) {
 	global $nfdump_date_format;
 	global $config;
 
 	// If there is aggregation calculate the top n
-	$sources = array ();
 	if ($aggregate != 'none') {
-		$agg_command = $command . " -s $aggregate -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
+		$values['data'] = array ();
+		$values['sources'] = array ();
+		$agg_command = $command . " -s $aggregate/$unit -n $max -t ".date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
 		exec($agg_command, $string);
 		foreach($string as $line){
 			if ($line=='') {
@@ -331,131 +333,35 @@ function netflow_get_data ($start_date, $end_date, $command, $aggregate, $max, $
 			$line = preg_replace('/\(\s*\S+\)/','',$line);
 			$line = preg_replace('/\s+/',' ',$line);
 			$val = explode(' ',$line);
-			$sources[$val[4]] = 1;
+			$values['sources'][$val[4]] = 1;
 		}
+	} else {
+		$values = array ();
 	}
 
-	// Execute nfdump and save its output in a temporary file
-	$command .= ' -t '.date($nfdump_date_format, $start_date).'-'.date($nfdump_date_format, $end_date);
-	$temp_file = $config["attachment_store"] . "/netflow_" . rand (0, getrandmax()) . ".data";
-	exec("$command > $temp_file");
+	// Load cache
+	$cache_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.cache';
+	$last_timestamp = netflow_load_cache ($values, $cache_file, $start_date, $end_date, $aggregate);
+	if ($last_timestamp < $end_date) {
 
-	// Open the temporary file
-	$fh = fopen ($temp_file, "r");
-	if ($fh === FALSE) {
-		return;
-	}
+		$last_timestamp++;
 
-	// Calculate the number of intervals
-	$num_intervals = $config['graph_res'] * 50;
-	$period = $end_date - $start_date;
-	$interval_length = (int) ($period / $num_intervals);
+		// Execute nfdump and save its output in a temporary file
+		$temp_file = $config['attachment_store'] . '/netflow_' . $unique_id . '.tmp';
+		$command .= ' -t '.date($nfdump_date_format, $last_timestamp).'-'.date($nfdump_date_format, $end_date);
+		exec("$command > $temp_file");
 
-	// Parse flow data
-	$read_flag = 1;
-	$values = array ();
-	$flow = array ();
-	for ($i = 0; $i < $num_intervals; $i++) {
-		$timestamp = $start_date + ($interval_length * $i);
+		// Parse data file
+		// We must parse from $start_date to avoid creating new intervals!
+		netflow_parse_file ($start_date, $end_date, $temp_file, $values, $aggregate, $unit);
 		
-		if ($aggregate != 'none') {
-			$interval_total = array ();
-			$interval_count = array ();
-		} else {
-			$interval_total = 0;
-			$interval_count = 0;
-		}
-
-		do {
-			if ($read_flag == 1) {
-				$read_flag = 0;
-				$line = fgets($fh, 4096);
-				if ($line === false) {
-					$read_flag = 1;
-					break;
-				}
-				
-				$line = preg_replace('/\s+/',' ',$line);
-				$val = explode(' ',$line);
-				if (! isset ($val[6])) {
-					$read_flag = 1;
-					break;
-				}
-				
-				$flow['date'] = $val[0];
-				$flow['time'] = $val[1];
-				
-				switch ($aggregate){
-					case "proto":
-						$flow['agg'] = $val[3];
-						break;
-					case "srcip":
-						$val2 = explode(':', $val[4]);
-						$flow['agg'] = $val2[0];
-						break;
-					case "srcport":
-						$val2 = explode(':', $val[4]);
-						$flow['agg'] = $val2[1];
-						break;
-					case "dstip":
-						$val2 = explode(':', $val[6]);
-						$flow['agg'] = $val2[0];
-						break;
-					case "dstport":
-						$val2 = explode(':', $val[6]);
-						$flow['agg'] = $val2[1];
-						break;
-				}
-				
-				switch ($unit) {
-					case "packets":
-						$flow['data'] = $val[7];
-						break;
-					case "bytes":
-						$flow['data'] = $val[8];
-						break;
-					case "flows":
-						$flow['data'] = $val[9];
-						break;
-				}
-				$flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']);
-			}
-			if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) {
-				$read_flag = 1;
-				if ($aggregate != 'none') {
-					if (isset ($sources[$flow['agg']])) {
-						if (! isset ($interval_total[$flow['agg']])) {
-							$interval_total[$flow['agg']] = 0;
-							$interval_count[$flow['agg']] = 0;
-						}
-						$interval_total[$flow['agg']] += $flow['data'];
-						$interval_count[$flow['agg']] += 1;
-					}
-				} else {
-					$interval_total += $flow['data'];
-					$interval_count += 1;
-				}
-			}
-		} while ($read_flag == 1);
-		
-		if ($aggregate != 'none') {
-			foreach ($interval_total as $agg => $val) {
-				if ($interval_count[$agg] != 0) {
-					$values['data'][$timestamp][$agg] = (int) ($interval_total[$agg] / $interval_count[$agg]);
-					$values['sources'][$agg] = 1;
-				}
-			}
-		} else {
-			if ($interval_count == 0) {
-				$values[$timestamp]['data'] = 0;
-			} else {
-				$values[$timestamp]['data'] = (int) ($interval_total / $interval_count);
-			}
-		}
+		unlink ($temp_file);
 	}
 
-	fclose ($fh);
-	unlink ($temp_file);
+	// Save cache
+	if ($aggregate == 'none') {
+		netflow_save_cache ($values, $cache_file);
+	}
 
 	return $values;
 }
@@ -619,4 +525,256 @@ function netflow_get_command ($filter) {
 	return $command;
 }
 
+
+/**
+ * Parses netflow data from the given file.
+ *
+ * @param string start_date Period start date.
+ * @param string end_date Period end date.
+ * @param string file File that contains netflow data.
+ * @param array values Array where netflow data will be placed.
+ * @param string aggregate Aggregate field.
+ * @param string unit Unit to show.
+ *
+ * @return Timestamp of the last data read.
+ *
+ */
+function netflow_parse_file ($start_date, $end_date, $file, &$values, $aggregate, $unit) {
+	global $config;
+
+	// Last timestamp read
+	$last_timestamp = $start_date;
+
+	// Open the data file
+	$fh = @fopen ($file, "r");
+	if ($fh === FALSE) {
+		return $last_timestamp;
+	}
+
+	// Calculate the number of intervals
+	$num_intervals = $config['graph_res'] * 50;
+	$period = $end_date - $start_date;
+	$interval_length = (int) ($period / $num_intervals);
+
+	// Parse flow data
+	$read_flag = 1;
+	$flow = array ();
+	for ($i = 0; $i < $num_intervals; $i++) {
+		$timestamp = $start_date + ($interval_length * $i);
+		
+		if ($aggregate != 'none') {
+			$interval_total = array ();
+			$interval_count = array ();
+		} else {
+			$interval_total = 0;
+			$interval_count = 0;
+		}
+
+		do {
+			if ($read_flag == 1) {
+				$read_flag = 0;
+				$line = fgets($fh, 4096);
+				if ($line === false) {
+					$read_flag = 1;
+					break;
+				}
+				
+				$line = preg_replace('/\s+/',' ',$line);
+				$val = explode(' ',$line);
+				if (! isset ($val[6])) {
+					$read_flag = 1;
+					break;
+				}
+				
+				$flow['date'] = $val[0];
+				$flow['time'] = $val[1];
+				
+				switch ($aggregate){
+					case "proto":
+						$flow['agg'] = $val[3];
+						break;
+					case "srcip":
+						$val2 = explode(':', $val[4]);
+						$flow['agg'] = $val2[0];
+						break;
+					case "srcport":
+						$val2 = explode(':', $val[4]);
+						$flow['agg'] = $val2[1];
+						break;
+					case "dstip":
+						$val2 = explode(':', $val[6]);
+						$flow['agg'] = $val2[0];
+						break;
+					case "dstport":
+						$val2 = explode(':', $val[6]);
+						$flow['agg'] = $val2[1];
+						break;
+				}
+				
+				switch ($unit) {
+					case "packets":
+						$flow['data'] = $val[7];
+						break;
+					case "bytes":
+						$flow['data'] = $val[8];
+						break;
+					case "flows":
+						$flow['data'] = $val[9];
+						break;
+				}
+				$flow['timestamp'] = strtotime ($flow['date'] . " " . $flow['time']);
+				$last_timestamp = $flow['timestamp'];
+			}
+			if ($flow['timestamp'] >= $timestamp && $flow['timestamp'] <= $timestamp + $interval_length) {
+				$read_flag = 1;
+				if ($aggregate != 'none') {
+					if (isset ($values['sources'][$flow['agg']])) {
+						if (! isset ($interval_total[$flow['agg']])) {
+							$interval_total[$flow['agg']] = 0;
+							$interval_count[$flow['agg']] = 0;
+						}
+						$interval_total[$flow['agg']] += $flow['data'];
+						$interval_count[$flow['agg']] += 1;
+					}
+				} else {
+					$interval_total += $flow['data'];
+					$interval_count += 1;
+				}
+			}
+		} while ($read_flag == 1);
+		
+		if ($aggregate != 'none') {
+			foreach ($interval_total as $agg => $val) {
+				
+				// No data for this interval/aggregate
+				if ($interval_count[$agg] == 0) {
+					continue;
+				}
+
+				// Read previous data for this interval
+				if (isset ($values['data'][$timestamp][$agg])) {
+					$previous_value = $values['data'][$timestamp][$agg];
+				} else {
+					$previous_value = 0;
+				}
+
+				// Calculate interval data
+				$values['data'][$timestamp][$agg] = (int) ($interval_total[$agg] / $interval_count[$agg]);
+			
+				// Average with previous data
+				if ($previous_value != 0) {
+					$values['data'][$timestamp][$agg] = (int) (($values['data'][$timestamp][$agg] + $previous_data) / 2);
+				}
+			}
+		} else {
+
+			// No data for this interval
+			if ($interval_count == 0) {
+				continue;
+			}
+			
+			// Read previous data for this interval
+			if (isset ($values[$timestamp]['data'])) {
+				$previous_value = $values[$timestamp]['data'];
+			} else {
+				$previous_value = 0;
+			}
+
+			// Calculate interval data
+			$values[$timestamp]['data'] = (int) ($interval_total / $interval_count);
+			
+			// Average with previous data
+			if ($previous_value != 0) {
+				$values[$timestamp]['data'] = (int) (($values[$timestamp]['data'] + $previous_value) / 2);
+			}
+
+			
+		}
+	}
+
+	fclose ($fh);
+	return $last_timestamp;
+}
+
+/**
+ * Save data to the specified cache file.
+ *
+ * @param array data Data array.
+ * @param string cache_file Cache file name.
+ *
+ */
+function netflow_save_cache ($data, $cache_file) {
+
+	@file_put_contents ($cache_file, serialize ($data));
+
+	return;
+}
+	
+/**
+ * Load data from the specified cache file.
+ *
+ * @param string data Array were cache data will be stored.
+ * @param string cache_file Cache file name.
+ * @param string start_date Period start date.
+ * @param string end_date Period end date.
+ *
+ * @return Timestamp of the last data read from cache.
+ *
+ */
+function netflow_load_cache (&$data, $cache_file, $start_date, $end_date, $aggregate) {
+	global $config;
+
+	// Open cache file
+	$cache_data = @file_get_contents ($cache_file);
+	$cache_data = @unserialize ($cache_data);
+
+	// Calculate the number of intervals
+	$num_intervals = $config['graph_res'] * 50;
+	$period = $end_date - $start_date;
+	$interval_length = (int) ($period / $num_intervals);
+	$last_timestamp = $start_date;
+
+	// Initializa chart data
+	if ($aggregate == 'none') {
+		if ($cache_data === FALSE) {
+			$cache_data = array ();
+		}
+		for ($i = 0; $i < $num_intervals; $i++) {
+			$timestamp = $start_date + ($interval_length * $i);
+			$interval_count = 0;
+			$interval_total = 0;
+			foreach ($cache_data as $cache_timestamp => $cache_value) {
+				if ($cache_timestamp < $timestamp + $interval_length) {
+					if ($cache_timestamp >= $timestamp) {
+						$interval_count++;
+						$interval_total += $cache_value['data'];
+						$last_timestamp = $cache_timestamp;
+					}
+					unset ($cache_data[$cache_timestamp]);
+				} else {
+					break;
+				}
+			}
+	
+			if ($interval_count > 0) {
+				$data[$timestamp]['data'] = (int) ($interval_total / $interval_count);
+			} else {
+				$data[$timestamp]['data'] = 0;
+			}
+		}
+	} else {
+               for ($i = 0; $i < $num_intervals; $i++) {
+                        $timestamp = $start_date + ($interval_length * $i);
+                        $interval_count = array ();
+                        $interval_total = array ();
+
+                        foreach ($data['sources'] as $source => $null) {
+                                $data['data'][$timestamp][$source] = 0;
+                        }
+                }
+	}
+
+	return $last_timestamp;
+}
+
 ?>
diff --git a/pandora_console/operation/netflow/nf_view.php b/pandora_console/operation/netflow/nf_view.php
index f38e8bbb2f..4889cc5b61 100644
--- a/pandora_console/operation/netflow/nf_view.php
+++ b/pandora_console/operation/netflow/nf_view.php
@@ -127,7 +127,9 @@ if (empty ($all_rcs)) {
 for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) {
 
 	// Get report item
-	$content_report = db_get_row_sql("SELECT * FROM tnetflow_report_content WHERE id_rc='" . $all_rcs[$x]['id_rc'] . "'");
+	$report_id = $all_rcs[$x]['id_rc'];
+	$content_report = db_get_row_sql("SELECT * FROM tnetflow_report_content WHERE id_rc='$report_id'");
+	$content_id = $content_report['id_rc'];
 	$max_aggregates= $content_report['max'];
 	$type = $content_report['show_graph'];
 	
@@ -140,7 +142,8 @@ for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) {
 	// Process item
 	switch ($type){
 		case '0':
-			$data = netflow_get_data ($start_date, $end_date, $command, $aggregate, $max_aggregates, $unit);
+			$unique_id = $report_id . '_' . $content_id . '_' . ($end_date - $start_date);
+			$data = netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max_aggregates, $unit);
 			if ($aggregate != 'none') {
 				echo graph_netflow_aggregate_area($data, $interval, 660, 320, '', '', '', '', $end_date, $unit);
 			} else {
@@ -152,7 +155,8 @@ for ($x = 0; isset($all_rcs[$x]['id_rc']); $x++) {
 			echo graph_netflow_aggregate_pie($result);
 			break;
 		case '2':
-			$data = netflow_get_data ($start_date, $end_date, $command, $aggregate, $max_aggregates, $unit);
+			$unique_id = $report_id . '_' . $content_id . '_' . ($end_date - $start_date);
+			$data = netflow_get_data ($start_date, $end_date, $command, $unique_id, $aggregate, $max_aggregates, $unit);
 			echo netflow_data_table ($data, $start_date, $end_date, $unit);
 			break;
 		case '3':