Added support to suppress unparseable messages when using msgparsers.

This enhances useability if you have mixed data sources, and want to
filter out unusefull messages.

src/admin/sources.php

@@ -70,7 +70,9 @@ if ( isset($_GET['op']) )
 		CreateSourceTypesList($content['SourceType']);
 		$content['MsgParserList'] = "";
 		$content['MsgNormalize'] = 0;
+		$content['MsgSkipUnparseable'] = 0;
 		$content['CHECKED_ISNORMALIZEMSG'] = "";
+		$content['CHECKED_ISSKIPUNPARSEABLE'] = "";
 
 		// Init View List!
 		$content['SourceViewID'] = 'SYSLOG';
@@ -143,6 +145,13 @@ if ( isset($_GET['op']) )
 				else
 					$content['CHECKED_ISNORMALIZEMSG'] = "";
 
+				$content['MsgSkipUnparseable'] = $mysource['MsgSkipUnparseable'];
+				if ( $mysource['MsgSkipUnparseable'] == 1 )
+					$content['CHECKED_ISSKIPUNPARSEABLE'] = "checked";
+				else
+					$content['CHECKED_ISSKIPUNPARSEABLE'] = "";
+				
+
 				// Init View List!
 				$content['SourceViewID'] = $mysource['ViewID'];
 				$content['VIEWS'] = $content['Views'];
@@ -273,6 +282,7 @@ if ( isset($_POST['op']) )
 	if ( isset($_POST['SourceType']) ) { $content['SourceType'] = DB_RemoveBadChars($_POST['SourceType']); }
 	if ( isset($_POST['MsgParserList']) ) { $content['MsgParserList'] = DB_RemoveBadChars($_POST['MsgParserList']); }
 	if ( isset($_POST['MsgNormalize']) ) { $content['MsgNormalize'] = intval(DB_RemoveBadChars($_POST['MsgNormalize'])); } else {$content['MsgNormalize'] = 0; }
+	if ( isset($_POST['MsgSkipUnparseable']) ) { $content['MsgSkipUnparseable'] = intval(DB_RemoveBadChars($_POST['MsgSkipUnparseable'])); } else {$content['MsgSkipUnparseable'] = 0; }
 	if ( isset($_POST['SourceViewID']) ) { $content['SourceViewID'] = DB_RemoveBadChars($_POST['SourceViewID']); }
 
 	if ( isset($content['SourceType']) )
@@ -426,6 +436,7 @@ if ( isset($_POST['op']) )
 		$tmpSource['SourceType']		= $content['SourceType'];
 		$tmpSource['MsgParserList']		= $content['MsgParserList'];
 		$tmpSource['MsgNormalize']		= $content['MsgNormalize'];
+		$tmpSource['MsgSkipUnparseable']= $content['MsgSkipUnparseable'];
 		$tmpSource['ViewID']			= $content['SourceViewID'];
 		if ( $tmpSource['SourceType'] == SOURCE_DISK ) 
 		{
@@ -473,12 +484,13 @@ if ( isset($_POST['op']) )
 			// Add custom search now!
 			if ( $content['SourceType'] == SOURCE_DISK ) 
 			{
-				$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, ViewID, LogLineType, DiskFile, userid, groupid) 
+				$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, LogLineType, DiskFile, userid, groupid) 
 				VALUES ('" . $content['Name'] . "', 
 						'" . $content['Description'] . "',
 						" . $content['SourceType'] . ", 
 						'" . $content['MsgParserList'] . "',
 						" . $content['MsgNormalize'] . ", 
+						" . $content['MsgSkipUnparseable'] . ", 
 						'" . $content['SourceViewID'] . "',
 						'" . $content['SourceLogLineType'] . "',
 						'" . $content['SourceDiskFile'] . "',
@@ -488,12 +500,13 @@ if ( isset($_POST['op']) )
 			}
 			else if ( $content['SourceType'] == SOURCE_DB || $content['SourceType'] == SOURCE_PDO ) 
 			{
-				$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, userid, groupid) 
+				$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, userid, groupid) 
 				VALUES ('" . $content['Name'] . "', 
 						'" . $content['Description'] . "',
 						" . $content['SourceType'] . ", 
 						'" . $content['MsgParserList'] . "', 
 						" . $content['MsgNormalize'] . ", 
+						" . $content['MsgSkipUnparseable'] . ", 
 						'" . $content['SourceViewID'] . "',
 						'" . $content['SourceDBTableType'] . "',
 						" . $content['SourceDBType'] . ",
@@ -534,6 +547,7 @@ if ( isset($_POST['op']) )
 									SourceType = " . $content['SourceType'] . ", 
 									MsgParserList = '" . $content['MsgParserList'] . "', 
 									MsgNormalize = " . $content['MsgNormalize'] . ", 
+									MsgSkipUnparseable = " . $content['MsgSkipUnparseable'] . ", 
 									ViewID = '" . $content['SourceViewID'] . "', 
 									LogLineType = '" . $content['SourceLogLineType'] . "', 
 									DiskFile = '" . $content['SourceDiskFile'] . "', 
@@ -549,6 +563,7 @@ if ( isset($_POST['op']) )
 									SourceType = " . $content['SourceType'] . ", 
 									MsgParserList = '" . $content['MsgParserList'] . "', 
 									MsgNormalize = " . $content['MsgNormalize'] . ", 
+									MsgSkipUnparseable = " . $content['MsgSkipUnparseable'] . ", 
 									ViewID = '" . $content['SourceViewID'] . "', 
 									DBTableType = '" . $content['SourceDBTableType'] . "', 
 									DBType = " . $content['SourceDBType'] . ", 

src/classes/logstreamconfig.class.php

@@ -53,6 +53,7 @@ abstract class LogStreamConfig {
 	protected $_msgParserList = null;		// Contains a string list of configure msg parsers
 	protected $_msgParserObjList = null;	// Contains an object reference list to the msg parsers
 	protected $_MsgNormalize = 0;			// If set to one, the msg will be reconstructed if successfully parsed before
+	public $_SkipUnparseable = 0;			// If set to one, all unparseable message will be ignored! This of course only applies if a msg parser is used
 
 	// Constructor prototype 
 	public abstract function LogStreamFactory($o);
@@ -79,6 +80,17 @@ abstract class LogStreamConfig {
 		}
 	}
 
+	/*
+	* Helper function to init Parserlist
+	*/
+	public function SetSkipUnparseable( $nNewVal )
+	{
+		if ( $nNewVal == 0 ) 
+			$this->_SkipUnparseable = 0;
+		else
+			$this->_SkipUnparseable = 1;
+	}
+
 	/*
 	* Helper function to init Parserlist
 	*/
@@ -147,6 +159,10 @@ abstract class LogStreamConfig {
 				$ret = $myMsgParser->ParseMsg($szMsg, $arrArguments);
 				if ( $ret == SUCCESS || $ret == ERROR_MSG_SKIPMESSAGE )
 					return $ret;
+					
+				// Extra check, if user wants to, we SKIP the message!
+				if ( $this->_SkipUnparseable == 1 && $ret == ERROR_MSG_NOMATCH )
+					return ERROR_MSG_SKIPMESSAGE;
 			}
 		}
 

src/classes/logstreamdb.class.php

@@ -235,10 +235,12 @@ class LogStreamDB extends LogStream {
 					// Now read new ones
 					$ret = $this->ReadNextRecordsFromDB($uID);
 
+					// Check if we found more records
 					if ( !isset($this->bufferedRecords[$this->_currentRecordNum] ) )
 						$ret = ERROR_NOMORERECORDS;
 				}
 			}
+
 			if ( $ret == SUCCESS )
 			{
 				// Init and set variables
@@ -265,7 +267,13 @@ class LogStreamDB extends LogStream {
 
 				// Run optional Message Parsers now
 				if ( isset($arrProperitesOut[SYSLOG_MESSAGE]) ) 
-					$this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
+				{
+					$retParser = $this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
+
+					// Check if we have to skip the message!
+					if ( $retParser == ERROR_MSG_SKIPMESSAGE )
+						$ret = $retParser;
+				}
 
 				// Set uID to the PropertiesOut! //DEBUG -> $this->_currentRecordNum;
 				$uID = $arrProperitesOut[SYSLOG_UID] = $this->bufferedRecords[$this->_currentRecordNum][$dbmapping[$szTableType][SYSLOG_UID]];
@@ -339,6 +347,8 @@ class LogStreamDB extends LogStream {
 						$totalpages = 1;
 
 					while( $bFound == false && $this->ReadNextIDsFromDB() == SUCCESS )
+					{
+						if ( isset($this->bufferedRecords) ) 
 						{
 							foreach ( $this->bufferedRecords as $myRecord )
 							{
@@ -359,6 +369,13 @@ class LogStreamDB extends LogStream {
 								if ( $this->_currentRecordNum % $this->_logStreamConfigObj->_pageCount == 0 ) 
 									$this->_currentPageNumber++;
 							}
+						}
+						else
+						{
+							// Return error code in this case!
+							$this->ResetBufferedRecords();
+							$ret = ERROR_NOMORERECORDS;
+						}
 						
 						if ( $this->_currentPageNumber > 1 && $this->_readDirection == EnumReadDirection::Forward) 
 							$this->_currentPageNumber = $totalpages - $this->_currentPageNumber + 1;

src/classes/logstreampdo.class.php

@@ -297,7 +297,13 @@ class LogStreamPDO extends LogStream {
 
 				// Run optional Message Parsers now
 				if ( isset($arrProperitesOut[SYSLOG_MESSAGE]) ) 
-					$this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
+				{
+					$retParser = $this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
+
+					// Check if we have to skip the message!
+					if ( $retParser == ERROR_MSG_SKIPMESSAGE )
+						$ret = $retParser;
+				}
 
 				// Set uID to the PropertiesOut! //DEBUG -> $this->_currentRecordNum;
 				$uID = $arrProperitesOut[SYSLOG_UID] = $this->bufferedRecords[$this->_currentRecordNum][$dbmapping[$szTableType][SYSLOG_UID]];

src/include/db_update_v7.txt

@@ -1,4 +1,5 @@
 -- New Database Structure Updates
+ALTER TABLE `logcon_sources` ADD `MsgSkipUnparseable` BOOL NOT NULL DEFAULT '0' AFTER `MsgNormalize` ;
 
 -- Insert data
 

src/include/functions_config.php

@@ -80,6 +80,12 @@ function InitSource(&$mysource)
 			$content['Sources'][$iSourceID]['MsgNormalize'] = 0;
 		}
 
+		if ( !isset($mysource['MsgSkipUnparseable']) )
+		{
+			$CFG['Sources'][$iSourceID]['MsgSkipUnparseable'] = 0;
+			$content['Sources'][$iSourceID]['MsgSkipUnparseable'] = 0;
+		}
+
 		if ( !isset($mysource['Description']) )
 		{
 			$CFG['Sources'][$iSourceID]['Description'] = "";
@@ -185,6 +191,7 @@ function InitSource(&$mysource)
 		$mysource['ObjRef']->_pageCount = GetConfigSetting("ViewEntriesPerPage", 50);
 		$mysource['ObjRef']->SetMsgParserList( $mysource['MsgParserList'] );
 		$mysource['ObjRef']->SetMsgNormalize( $mysource['MsgNormalize'] );
+		$mysource['ObjRef']->SetSkipUnparseable( $mysource['MsgSkipUnparseable'] );
 
 		// Set default SourceID here!
 		if ( isset($content['Sources'][$iSourceID]) && !isset($currentSourceID) ) 

src/include/functions_db.php

@@ -45,7 +45,7 @@ $errdesc = "";
 $errno = 0;
 
 // --- Current Database Version, this is important for automated database Updates!
-$content['database_internalversion'] = "6";	// Whenever incremented, a database upgrade is needed
+$content['database_internalversion'] = "7";	// Whenever incremented, a database upgrade is needed
 $content['database_installedversion'] = "0";	// 0 is default which means Prior Versioning Database
 // --- 
 

src/index.php

@@ -267,7 +267,17 @@ if ( isset($content['Sources'][$currentSourceID]) )
 				}
 			}
 		}
-		else
+		else if ( $ret == ERROR_MSG_SKIPMESSAGE )
+		{
+			do
+			{
+				// Skip until we find a suitable entry
+				$ret = $stream->ReadNext($uID, $logArray);
+			} while ($ret == ERROR_MSG_SKIPMESSAGE);
+		}
+		
+		// check for error return state!
+		if ( $ret != SUCCESS )
 		{
 			// This will disable to Main SyslogView and show an error message
 			$content['syslogmessagesenabled'] = "false";
@@ -625,7 +635,14 @@ if ( isset($content['Sources'][$currentSourceID]) )
 
 				// Increment Counter
 				$counter++;
-			} while ($counter < $content['CurrentViewEntriesPerPage'] && ($ret = $stream->ReadNext($uID, $logArray)) == SUCCESS);
+				
+				// --- Extra Loop to get the next entry!
+				do
+				{
+					$ret = $stream->ReadNext($uID, $logArray);
+				} while ( $ret == ERROR_MSG_SKIPMESSAGE );
+				// --- 
+			} while ( $counter < $content['CurrentViewEntriesPerPage'] && ($ret == SUCCESS) );
 //print_r ( $content['syslogmessages'] );
 
 			// Move below processing - Read First and LAST UID's before start reading the stream!

src/lang/de/main.php

@@ -192,6 +192,7 @@ $content['LN_CFG_FIRSTSYSLOGSOURCE'] = "Erste Syslog Quelle";
 	$content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
 	$content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
 	$content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
+	$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
 
 // Details page
 $content['LN_DETAILS_FORSYSLOGMSG'] = "Details für syslog-Nachrichten mit der ID";

src/lang/en/main.php

@@ -194,6 +194,7 @@ $content['LN_CFG_VIEW'] = "Select View";
 $content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
 $content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
 $content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
+$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
 
 // Details page
 $content['LN_DETAILS_FORSYSLOGMSG'] = "Details for the syslog messages with id";

src/lang/pt_BR/main.php

@@ -196,6 +196,7 @@ $content['LN_CFG_VIEW'] = "Selecione visão";
 	$content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
 	$content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
 	$content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
+	$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
 
 // Details page
 $content['LN_DETAILS_FORSYSLOGMSG'] = "Detalhes para a mensagem com id";

src/templates/admin/admin_sources.html

@@ -135,6 +135,10 @@
 				<td align="left" class="cellmenu2" nowrap><b>{LN_CFG_NORMALIZEMSG}</b></td>
 				<td align="right" class="line1"><input type="checkbox" name="MsgNormalize" value="1" {CHECKED_ISNORMALIZEMSG}></td>
 			</tr>
+			<tr>
+				<td align="left" class="cellmenu2" nowrap><b>{LN_CFG_SKIPUNPARSEABLE}</b></td>
+				<td align="right" class="line1"><input type="checkbox" name="MsgSkipUnparseable" value="1" {CHECKED_ISSKIPUNPARSEABLE}></td>
+			</tr>
 
 			<tr>
 				<td align="left" class="cellmenu2"><b>{LN_GEN_USERONLY}</b></td>