2013-01-16 Sergio Martin <sergio.martin@artica.es>

* util/pandora_db.pl: Added the history events to the 
	maintenance script of database and delete the old events in
	1000 events blocks

	* util/pandora_manage.pl: Added a CLI function to 
	create new netflow filters




git-svn-id: https://svn.code.sf.net/p/pandora/code/trunk@7488 c3f86ba8-e40f-0410-aaad-9ba5e7f4b01f

pandora_server/ChangeLog

@@ -1,3 +1,12 @@
+2013-01-16  Sergio Martin <sergio.martin@artica.es>
+
+	* util/pandora_db.pl: Added the history events to the 
+	maintenance script of database and delete the old events in
+	1000 events blocks
+
+	* util/pandora_manage.pl: Added a CLI function to 
+	create new netflow filters
+
 2013-01-16  Vanessa Gil <vanessa.gil@artica.es>
 
 	* util/pandora_revent.pl: Added 'agent_name' and

pandora_server/util/pandora_db.pl

@@ -182,10 +182,47 @@ sub pandora_purgedb ($$) {
         $conf->{'_event_purge'}= 10;
     }
 
-	print "[PURGE] Deleting old event data (More than " . $conf->{'_event_purge'} . " days)... \n";
-
     my $event_limit = time() - 86400 * $conf->{'_event_purge'};
-	db_do($dbh, "DELETE FROM tevento WHERE utimestamp < $event_limit");
+    
+    my $events_table = 'tevento';
+    
+	# If is installed enterprise version and enabled metaconsole, 
+	# check the events history copy and set the name of the metaconsole events table
+    if (defined($conf->{'_enterprise_installed'}) && $conf->{'_enterprise_installed'} eq '1' &&
+		defined($conf->{'_metaconsole'}) && $conf->{'_metaconsole'} eq '1'){
+	
+		# If events history is enabled, save the new events (not validated or in process) to history database
+		if(defined($conf->{'_metaconsole_events_history'}) && $conf->{'_metaconsole_events_history'} eq '1') {
+			print "[PURGE] Moving old not validated events to history table (More than " . $conf->{'_event_purge'} . " days)... \n";
+
+			my @events = get_db_rows ($dbh, 'SELECT * FROM tmetaconsole_event WHERE estado = 0 AND utimestamp < ?', $event_limit);
+
+			foreach my $event (@events) {
+				db_process_insert($dbh, 'id_evento', 'tmetaconsole_event_history', $event);
+			}
+		}
+		
+		$events_table = 'tmetaconsole_event';
+	}
+	
+	print "[PURGE] Deleting old event data at $events_table table (More than " . $conf->{'_event_purge'} . " days)... \n";
+
+	# Delete with buffer to avoid problems with performance
+	my $buffer = 1000;
+	
+	my $events_to_delete = get_db_value ($dbh, "SELECT COUNT(*) FROM $events_table WHERE utimestamp < ?", $event_limit);
+
+	while(1) {
+		db_do($dbh, "DELETE FROM $events_table WHERE utimestamp < ? LIMIT ?", $event_limit, $buffer);
+
+		if($events_to_delete <= $buffer) {
+			last;
+		}
+		else {
+			$events_to_delete = $events_to_delete - $buffer;
+		}
+	}
+		
 
     # Delete audit data
     if (!defined($conf->{'_audit_purge'})){
@@ -433,6 +470,10 @@ sub pandora_load_config ($) {
 	$conf->{'_history_db_step'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'history_db_step'");
 	$conf->{'_history_db_delay'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'history_db_delay'");
 	$conf->{'_days_delete_unknown'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'days_delete_unknown'");
+	$conf->{'_enterprise_installed'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'enterprise_installed'");
+	$conf->{'_metaconsole'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'metaconsole'");
+	$conf->{'_metaconsole_events_history'} = get_db_value ($dbh, "SELECT value FROM tconfig WHERE token = 'metaconsole_events_history'");
+    
 	db_disconnect ($dbh);
 
 	printf "Pandora DB now initialized and running (PURGE=" . $conf->{'_days_purge'} . " days, COMPACT=$conf->{'_days_compact'} days, STEP=" . $conf->{'_step_compact'} . ") ... \n\n";

pandora_server/util/pandora_manage.pl

@@ -153,6 +153,8 @@ sub help_screen{
 	help_screen_line('--validate_policy_alerts', '<policy_name>', 'Validate the alerts of a given policy');
 	help_screen_line('--get_policy_modules', '<policy_name>', 'Get the modules of a policy');
 	help_screen_line('--get_policies', '[<agent_name>]', 'Get all the policies (without parameters) or the policies of a given agent (agent name as parameter)');
+	print "NETFLOW:\n\n" unless $param ne '';
+	help_screen_line('--create_netflow_filter', '<filter_name> <group_name> <filter> <aggregate_by dstip|dstport|none|proto|srcip|srcport> <output_format kilobytes|kilobytespersecond|megabytes|megabytespersecond>', 'Create a new netflow filter');
 	print "TOOLS:\n\n" unless $param ne '';
 	help_screen_line('--exec_from_file', '<file_path> <option_to_execute> <option_params>', 'Execute any CLI option with macros from CSV file');
 	
@@ -1034,6 +1036,26 @@ sub cli_create_network_module_from_component() {
 
 }
 
+##############################################################################
+# Create netflow filter
+# Related option: --create_netflow_filter
+##############################################################################
+
+sub cli_create_netflow_filter() {
+	my ($filter_name, $group_name, $filter, $aggregate_by, $output_format) = @ARGV[2..6];
+	
+	my $group_id = get_group_id($dbh, $group_name);
+	exist_check($group_id,'group',$group_name);
+	
+	logger($conf, 'Creating netflow filter "' . $filter_name . '"', 10);
+
+	# Create the module
+	my $module_id = db_insert ($dbh, 'id_sg', 'INSERT INTO tnetflow_filter (id_name, id_group, advanced_filter, filter_args, aggregate, output)
+												VALUES (?, ?, ?, ?, ?, ?)',
+												safe_input($filter_name), $group_id, safe_input($filter), 
+												'"(' . $filter . ')"', $aggregate_by, $output_format);
+}
+
 ##############################################################################
 # Create network module.
 # Related option: --create_network_module
@@ -3498,6 +3520,10 @@ sub pandora_manage_main ($$$) {
 			param_check($ltotal, 2);
 			cli_create_network_module_from_component();
 		}
+		elsif ($param eq '--create_netflow_filter') {
+			param_check($ltotal, 5);
+			cli_create_netflow_filter();
+		}
 		else {
 			print_log "[ERROR] Invalid option '$param'.\n\n";
 			$param = '';