tyler.durden/loganalyzer
1
0
Fork 0
mirror of https://github.com/rsyslog/loganalyzer.git synced 2025-09-23 09:58:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added support to suppress unparseable messages when using msgparsers.

Browse Source 
This enhances useability if you have mixed data sources, and want to
filter out unusefull messages.
This commit is contained in:
Andre Lorbach 2008-10-27 16:18:42 +01:00
parent c4a8889a09
commit 5a22843551
13 changed files with 117 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/admin/sources.php
View File

@ -70,7 +70,9 @@ if ( isset($_GET['op']) )
CreateSourceTypesList($content['SourceType']);
$content['MsgParserList'] = "";
$content['MsgNormalize'] = 0;
$content['MsgSkipUnparseable'] = 0;
$content['CHECKED_ISNORMALIZEMSG'] = "";
$content['CHECKED_ISSKIPUNPARSEABLE'] = "";
// Init View List!
$content['SourceViewID'] = 'SYSLOG';
@ -143,6 +145,13 @@ if ( isset($_GET['op']) )
else
$content['CHECKED_ISNORMALIZEMSG'] = "";
$content['MsgSkipUnparseable'] = $mysource['MsgSkipUnparseable'];
if ( $mysource['MsgSkipUnparseable'] == 1 )
$content['CHECKED_ISSKIPUNPARSEABLE'] = "checked";
else
$content['CHECKED_ISSKIPUNPARSEABLE'] = "";
// Init View List!
$content['SourceViewID'] = $mysource['ViewID'];
$content['VIEWS'] = $content['Views'];
@ -273,6 +282,7 @@ if ( isset($_POST['op']) )
if ( isset($_POST['SourceType']) ) { $content['SourceType'] = DB_RemoveBadChars($_POST['SourceType']); }
if ( isset($_POST['MsgParserList']) ) { $content['MsgParserList'] = DB_RemoveBadChars($_POST['MsgParserList']); }
if ( isset($_POST['MsgNormalize']) ) { $content['MsgNormalize'] = intval(DB_RemoveBadChars($_POST['MsgNormalize'])); } else {$content['MsgNormalize'] = 0; }
if ( isset($_POST['MsgSkipUnparseable']) ) { $content['MsgSkipUnparseable'] = intval(DB_RemoveBadChars($_POST['MsgSkipUnparseable'])); } else {$content['MsgSkipUnparseable'] = 0; }
if ( isset($_POST['SourceViewID']) ) { $content['SourceViewID'] = DB_RemoveBadChars($_POST['SourceViewID']); }
if ( isset($content['SourceType']) )
@ -420,13 +430,14 @@ if ( isset($_POST['op']) )
include($gl_root_path . 'classes/logstream.class.php');
// First create a tmp source array
$tmpSource['ID'] = $content['SOURCEID'];
$tmpSource['Name'] = $content['Name'];
$tmpSource['Description'] = $content['Description'];
$tmpSource['SourceType'] = $content['SourceType'];
$tmpSource['MsgParserList'] = $content['MsgParserList'];
$tmpSource['MsgNormalize'] = $content['MsgNormalize'];
$tmpSource['ViewID'] = $content['SourceViewID'];
$tmpSource['ID'] = $content['SOURCEID'];
$tmpSource['Name'] = $content['Name'];
$tmpSource['Description'] = $content['Description'];
$tmpSource['SourceType'] = $content['SourceType'];
$tmpSource['MsgParserList'] = $content['MsgParserList'];
$tmpSource['MsgNormalize'] = $content['MsgNormalize'];
$tmpSource['MsgSkipUnparseable']= $content['MsgSkipUnparseable'];
$tmpSource['ViewID'] = $content['SourceViewID'];
if ( $tmpSource['SourceType'] == SOURCE_DISK )
{
$tmpSource['LogLineType'] = $content['SourceLogLineType'];
@ -473,12 +484,13 @@ if ( isset($_POST['op']) )
// Add custom search now!
if ( $content['SourceType'] == SOURCE_DISK )
{
$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, ViewID, LogLineType, DiskFile, userid, groupid)
$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, LogLineType, DiskFile, userid, groupid)
VALUES ('" . $content['Name'] . "',
'" . $content['Description'] . "',
" . $content['SourceType'] . ",
'" . $content['MsgParserList'] . "',
" . $content['MsgNormalize'] . ",
" . $content['MsgSkipUnparseable'] . ",
'" . $content['SourceViewID'] . "',
'" . $content['SourceLogLineType'] . "',
'" . $content['SourceDiskFile'] . "',
@ -488,12 +500,13 @@ if ( isset($_POST['op']) )
}
else if ( $content['SourceType'] == SOURCE_DB || $content['SourceType'] == SOURCE_PDO )
{
$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, userid, groupid)
$sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, userid, groupid)
VALUES ('" . $content['Name'] . "',
'" . $content['Description'] . "',
" . $content['SourceType'] . ",
'" . $content['MsgParserList'] . "',
" . $content['MsgNormalize'] . ",
" . $content['MsgSkipUnparseable'] . ",
'" . $content['SourceViewID'] . "',
'" . $content['SourceDBTableType'] . "',
" . $content['SourceDBType'] . ",
@ -534,6 +547,7 @@ if ( isset($_POST['op']) )
SourceType = " . $content['SourceType'] . ",
MsgParserList = '" . $content['MsgParserList'] . "',
MsgNormalize = " . $content['MsgNormalize'] . ",
MsgSkipUnparseable = " . $content['MsgSkipUnparseable'] . ",
ViewID = '" . $content['SourceViewID'] . "',
LogLineType = '" . $content['SourceLogLineType'] . "',
DiskFile = '" . $content['SourceDiskFile'] . "',
@ -549,6 +563,7 @@ if ( isset($_POST['op']) )
SourceType = " . $content['SourceType'] . ",
MsgParserList = '" . $content['MsgParserList'] . "',
MsgNormalize = " . $content['MsgNormalize'] . ",
MsgSkipUnparseable = " . $content['MsgSkipUnparseable'] . ",
ViewID = '" . $content['SourceViewID'] . "',
DBTableType = '" . $content['SourceDBTableType'] . "',
DBType = " . $content['SourceDBType'] . ",

18
src/classes/logstreamconfig.class.php
View File

@ -53,7 +53,8 @@ abstract class LogStreamConfig {
protected $_msgParserList = null; // Contains a string list of configure msg parsers
protected $_msgParserObjList = null; // Contains an object reference list to the msg parsers
protected $_MsgNormalize = 0; // If set to one, the msg will be reconstructed if successfully parsed before
public $_SkipUnparseable = 0; // If set to one, all unparseable message will be ignored! This of course only applies if a msg parser is used
// Constructor prototype
public abstract function LogStreamFactory($o);
@ -79,6 +80,17 @@ abstract class LogStreamConfig {
}
}
/*
* Helper function to init Parserlist
*/
public function SetSkipUnparseable( $nNewVal )
{
if ( $nNewVal == 0 )
$this->_SkipUnparseable = 0;
else
$this->_SkipUnparseable = 1;
}
/*
* Helper function to init Parserlist
*/
@ -147,6 +159,10 @@ abstract class LogStreamConfig {
$ret = $myMsgParser->ParseMsg($szMsg, $arrArguments);
if ( $ret == SUCCESS || $ret == ERROR_MSG_SKIPMESSAGE )
return $ret;
// Extra check, if user wants to, we SKIP the message!
if ( $this->_SkipUnparseable == 1 && $ret == ERROR_MSG_NOMATCH )
return ERROR_MSG_SKIPMESSAGE;
}
}

49
src/classes/logstreamdb.class.php
View File

@ -235,10 +235,12 @@ class LogStreamDB extends LogStream {
// Now read new ones
$ret = $this->ReadNextRecordsFromDB($uID);
// Check if we found more records
if ( !isset($this->bufferedRecords[$this->_currentRecordNum] ) )
$ret = ERROR_NOMORERECORDS;
}
}
if ( $ret == SUCCESS )
{
// Init and set variables
@ -265,7 +267,13 @@ class LogStreamDB extends LogStream {
// Run optional Message Parsers now
if ( isset($arrProperitesOut[SYSLOG_MESSAGE]) )
$this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
{
$retParser = $this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
// Check if we have to skip the message!
if ( $retParser == ERROR_MSG_SKIPMESSAGE )
$ret = $retParser;
}
// Set uID to the PropertiesOut! //DEBUG -> $this->_currentRecordNum;
$uID = $arrProperitesOut[SYSLOG_UID] = $this->bufferedRecords[$this->_currentRecordNum][$dbmapping[$szTableType][SYSLOG_UID]];
@ -340,24 +348,33 @@ class LogStreamDB extends LogStream {
while( $bFound == false && $this->ReadNextIDsFromDB() == SUCCESS )
{
foreach ( $this->bufferedRecords as $myRecord )
if ( isset($this->bufferedRecords) )
{
if ( $myRecord[$uidfieldname] == $uID )
foreach ( $this->bufferedRecords as $myRecord )
{
$bFound = true;
$ret = SUCCESS;
break; // Break foreach loop!
if ( $myRecord[$uidfieldname] == $uID )
{
$bFound = true;
$ret = SUCCESS;
break; // Break foreach loop!
}
else
{
$tmpuID = $myRecord[$uidfieldname];
// Only Increment $_currentRecordNum
$this->_currentRecordNum++;
}
// Increment our Pagenumber if needed!
if ( $this->_currentRecordNum % $this->_logStreamConfigObj->_pageCount == 0 )
$this->_currentPageNumber++;
}
else
{
$tmpuID = $myRecord[$uidfieldname];
// Only Increment $_currentRecordNum
$this->_currentRecordNum++;
}
// Increment our Pagenumber if needed!
if ( $this->_currentRecordNum % $this->_logStreamConfigObj->_pageCount == 0 )
$this->_currentPageNumber++;
}
else
{
// Return error code in this case!
$this->ResetBufferedRecords();
$ret = ERROR_NOMORERECORDS;
}
if ( $this->_currentPageNumber > 1 && $this->_readDirection == EnumReadDirection::Forward)

2
src/classes/logstreamdisk.class.php
View File

@ -247,7 +247,7 @@ class LogStreamDisk extends LogStream {
// Check if we have to skip the message!
if ( $retParser == ERROR_MSG_SKIPMESSAGE )
$ret = $retParser;
// Set uID to the PropertiesOut!
$arrProperitesOut[SYSLOG_UID] = $uID;
}

8
src/classes/logstreampdo.class.php
View File

@ -297,7 +297,13 @@ class LogStreamPDO extends LogStream {
// Run optional Message Parsers now
if ( isset($arrProperitesOut[SYSLOG_MESSAGE]) )
$this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
{
$retParser = $this->_logStreamConfigObj->ProcessMsgParsers($arrProperitesOut[SYSLOG_MESSAGE], $arrProperitesOut);
// Check if we have to skip the message!
if ( $retParser == ERROR_MSG_SKIPMESSAGE )
$ret = $retParser;
}
// Set uID to the PropertiesOut! //DEBUG -> $this->_currentRecordNum;
$uID = $arrProperitesOut[SYSLOG_UID] = $this->bufferedRecords[$this->_currentRecordNum][$dbmapping[$szTableType][SYSLOG_UID]];

1
src/include/db_update_v7.txt
View File

@ -1,4 +1,5 @@
-- New Database Structure Updates
ALTER TABLE `logcon_sources` ADD `MsgSkipUnparseable` BOOL NOT NULL DEFAULT '0' AFTER `MsgNormalize` ;
-- Insert data

7
src/include/functions_config.php
View File

@ -80,6 +80,12 @@ function InitSource(&$mysource)
$content['Sources'][$iSourceID]['MsgNormalize'] = 0;
}
if ( !isset($mysource['MsgSkipUnparseable']) )
{
$CFG['Sources'][$iSourceID]['MsgSkipUnparseable'] = 0;
$content['Sources'][$iSourceID]['MsgSkipUnparseable'] = 0;
}
if ( !isset($mysource['Description']) )
{
$CFG['Sources'][$iSourceID]['Description'] = "";
@ -185,6 +191,7 @@ function InitSource(&$mysource)
$mysource['ObjRef']->_pageCount = GetConfigSetting("ViewEntriesPerPage", 50);
$mysource['ObjRef']->SetMsgParserList( $mysource['MsgParserList'] );
$mysource['ObjRef']->SetMsgNormalize( $mysource['MsgNormalize'] );
$mysource['ObjRef']->SetSkipUnparseable( $mysource['MsgSkipUnparseable'] );
// Set default SourceID here!
if ( isset($content['Sources'][$iSourceID]) && !isset($currentSourceID) )

2
src/include/functions_db.php
View File

@ -45,7 +45,7 @@ $errdesc = "";
$errno = 0;
// --- Current Database Version, this is important for automated database Updates!
$content['database_internalversion'] = "6"; // Whenever incremented, a database upgrade is needed
$content['database_internalversion'] = "7"; // Whenever incremented, a database upgrade is needed
$content['database_installedversion'] = "0"; // 0 is default which means Prior Versioning Database
// ---

21
src/index.php
View File

@ -267,7 +267,17 @@ if ( isset($content['Sources'][$currentSourceID]) )
}
}
}
else
else if ( $ret == ERROR_MSG_SKIPMESSAGE )
{
do
{
// Skip until we find a suitable entry
$ret = $stream->ReadNext($uID, $logArray);
} while ($ret == ERROR_MSG_SKIPMESSAGE);
}
// check for error return state!
if ( $ret != SUCCESS )
{
// This will disable to Main SyslogView and show an error message
$content['syslogmessagesenabled'] = "false";
@ -625,7 +635,14 @@ if ( isset($content['Sources'][$currentSourceID]) )
// Increment Counter
$counter++;
} while ($counter < $content['CurrentViewEntriesPerPage'] && ($ret = $stream->ReadNext($uID, $logArray)) == SUCCESS);
// --- Extra Loop to get the next entry!
do
{
$ret = $stream->ReadNext($uID, $logArray);
} while ( $ret == ERROR_MSG_SKIPMESSAGE );
// ---
} while ( $counter < $content['CurrentViewEntriesPerPage'] && ($ret == SUCCESS) );
//print_r ( $content['syslogmessages'] );
// Move below processing - Read First and LAST UID's before start reading the stream!

1
src/lang/de/main.php
View File

@ -192,6 +192,7 @@ $content['LN_CFG_FIRSTSYSLOGSOURCE'] = "Erste Syslog Quelle";
$content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
$content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
$content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
// Details page
$content['LN_DETAILS_FORSYSLOGMSG'] = "Details für syslog-Nachrichten mit der ID";

1
src/lang/en/main.php
View File

@ -194,6 +194,7 @@ $content['LN_CFG_VIEW'] = "Select View";
$content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
$content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
$content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
// Details page
$content['LN_DETAILS_FORSYSLOGMSG'] = "Details for the syslog messages with id";

1
src/lang/pt_BR/main.php
View File

@ -196,6 +196,7 @@ $content['LN_CFG_VIEW'] = "Selecione vis&atilde;o";
$content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in";
$content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)";
$content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers";
$content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)";
// Details page
$content['LN_DETAILS_FORSYSLOGMSG'] = "Detalhes para a mensagem com id";

4
src/templates/admin/admin_sources.html
View File

@ -135,6 +135,10 @@
<td align="left" class="cellmenu2" nowrap><b>{LN_CFG_NORMALIZEMSG}</b></td>
<td align="right" class="line1"><input type="checkbox" name="MsgNormalize" value="1" {CHECKED_ISNORMALIZEMSG}></td>
</tr>
<tr>
<td align="left" class="cellmenu2" nowrap><b>{LN_CFG_SKIPUNPARSEABLE}</b></td>
<td align="right" class="line1"><input type="checkbox" name="MsgSkipUnparseable" value="1" {CHECKED_ISSKIPUNPARSEABLE}></td>
</tr>
<tr>
<td align="left" class="cellmenu2"><b>{LN_GEN_USERONLY}</b></td>