Added support to show dynamic fields for MongoDB logstream sources.

This enables Loganalyzer to view data prior written by rsyslog into MongoDB
that came from CEE sources.

src/classes/logstream.class.php

@@ -473,14 +473,10 @@ abstract class LogStream {
 									{
 										// Include Filter
 										if ( $myfilter[FILTER_MODE] & FILTER_MODE_INCLUDE ) 
-										{
-											// Unless REGEX Filter, this has to be done by the Logstream driver
-											if ( !($myfilter[FILTER_MODE] & FILTER_MODE_SEARCHREGEX) )
 										{
 											if ( stripos($propertyvalue, $myfilter[FILTER_VALUE]) === false ) 
 												$bEval = false;
 										}
-										}
 										// Exclude Filter
 										else if ( $myfilter[FILTER_MODE] & FILTER_MODE_EXCLUDE ) 
 										{
@@ -718,6 +714,23 @@ abstract class LogStream {
 		return $szFieldName; 
 	}
 
+	/*
+	*	Helper function to check a if a fieldname exists in the mapping
+	*/
+	public function CheckFieldnameInMapping($szTableType, $szFieldName)
+	{
+		global $content, $dbmapping;
+
+		foreach( $dbmapping[$szTableType]['DBMAPPINGS'] as $myFieldID => $myDBMapping ) 
+		{
+			if ( $myDBMapping == $szFieldName ) 
+				return true; // return found!
+		}
+
+		// Default FALSE! 
+		return false; 
+	}
+
 	/*
 	* --- PIRVATE HELPERS!
 	*/

src/classes/logstreammongodb.class.php

@@ -488,8 +488,25 @@ class LogStreamMongoDB extends LogStream {
 							$arrProperitesOut[$property] = '';
 					}
 					else
+					{
 						$arrProperitesOut[$property] = '';
+//						echo $property . "=" . $this->bufferedRecords[$this->_currentRecordNum][$dbfieldname];
 					}
+				}
+				
+				// --- Add dynamic fields into record! 
+				foreach( $this->bufferedRecords[$this->_currentRecordNum] as $propName => $propValue)
+				{
+					if (	!isset($arrProperitesOut[$propName]) && 
+							!$this->CheckFieldnameInMapping($szTableType, $propName) && 
+							(isset($propValue) && strlen($propValue) > 0)
+						)
+					{	
+						// Add dynamic Property!
+						$arrProperitesOut[$propName] = 	$propValue; 
+					}
+				}
+				// --- 
 
 				// Run optional Message Parsers now
 				if ( isset($arrProperitesOut[SYSLOG_MESSAGE]) ) 
@@ -1572,7 +1589,12 @@ class LogStreamMongoDB extends LogStream {
 			OutputDebugMessage("LogStreamMongoDB|ReadNextRecordsFromDB: Running FIND ", DEBUG_ULTRADEBUG);
 			
 			// Find Data in MongoCollection 
-			$myCursor = $this->_myMongoCollection->find($this->_myMongoQuery, $this->_myMongoFields); 
+			$myCursor = $this->_myMongoCollection->find($this->_myMongoQuery)->limit($this->_logStreamConfigObj->RecordsPerQuery)->sort(array("_id" => -1)); // , $this->_myMongoFields); 
+
+//	echo "<pre>";
+//	var_dump(iterator_to_array($myCursor));
+//	echo "</pre>";
+
 		}
 		catch ( MongoCursorException $e ) 
 		{
@@ -1587,11 +1609,9 @@ class LogStreamMongoDB extends LogStream {
 		// OutputDebugMessage("LogStreamMongoDB|ReadNextRecordsFromDB: myCursor->info() = <pre>" . var_export($myCursor->info(), true) . "</pre>", DEBUG_ULTRADEBUG);
 
 		// Limit records
-		$myCursor->limit( $this->_logStreamConfigObj->RecordsPerQuery );
+//		$myCursor->limit( $this->_logStreamConfigObj->RecordsPerQuery );
-
 		// OutputDebugMessage("Cursor verbose: " . var_export($myCursor->explain(), true), DEBUG_DEBUG);
-		$myCursor = $myCursor->sort(array("_id" => -1));
+//		$myCursor = $myCursor->sort(array("_id" => -1));
-
 
 		try 
 		{
@@ -1621,6 +1641,7 @@ class LogStreamMongoDB extends LogStream {
 
 				// Keys will be converted into lowercase!
 				$this->bufferedRecords[$iBegin] = array_change_key_case( $myRow, CASE_LOWER);
+
 				$iBegin++;
 			}
 		}

src/details.php

@@ -208,7 +208,7 @@ if ( isset($content['Sources'][$currentSourceID]) ) // && $content['uid_current'
 					else
 						$content['fields'][$mycolkey]['cssclass'] = "line2";
 
-					if ( $mycolkey == SYSLOG_MESSAGE)
+					if ( $mycolkey == SYSLOG_MESSAGE )
 						$content['fields'][$mycolkey]['menucssclass'] = "cellmenu1_naked";
 					else
 						$content['fields'][$mycolkey]['menucssclass'] = "cellmenu1";
@@ -301,8 +301,35 @@ if ( isset($content['Sources'][$currentSourceID]) ) // && $content['uid_current'
 
 			}
 			
-//print_r ( $content['fields'] );
+			// --- Now Check for dynamic fields!
-//exit;
+			$counter = 0;
+			foreach($logArray as $mydynkey => $mydynvalue)
+			{
+				// Check if field is already in fields array
+				if (  !isset($content['fields'][$mydynkey]) && isset($mydynvalue) && strlen($mydynvalue) > 0 )
+				{
+					$content['dynamicfields'][$mydynkey]['dynfieldkey'] = $mydynkey;
+					$content['dynamicfields'][$mydynkey]['dynfieldvalue'] = $mydynvalue;
+					
+					// --- Set CSS Class
+					if ( $counter % 2 == 0 )
+						$content['dynamicfields'][$mydynkey]['dyncssclass'] = "line1";
+					else
+						$content['dynamicfields'][$mydynkey]['dyncssclass'] = "line2";
+					// ---
+
+					// Increment helpcounter
+					$counter++;
+				}
+			}
+			// Enable dynamic Fields
+			if ( isset($content['dynamicfields']) )
+				$content['dynamicfieldsenabled'] = "true";
+			// --- 
+
+//	echo "<pre>";
+//	var_dump($content['dynamicfields']);
+//	echo "</pre>";
 			
 			// Enable pager if the count is above 1 or we don't know the record count!
 			if ( $content['main_recordcount'] > 1 || $content['main_recordcount'] == -1 )

src/include/constants_logstream.php

@@ -388,6 +388,7 @@ $dbmapping['mongodb']['DBMAPPINGS'][SYSLOG_SEVERITY] = "syslog_sever";
 $dbmapping['mongodb']['DBMAPPINGS'][SYSLOG_SYSLOGTAG] = "procid"; // not using syslog_tag because of PID in it
 $dbmapping['mongodb']['DBMAPPINGS'][SYSLOG_PROCESSID] = "pid";
 $dbmapping['mongodb']['DBMAPPINGS'][MISC_CHECKSUM] = "Checksum";
+$dbmapping['mongodb']['DBMAPPINGS'][SYSLOG_EVENT_LOGTYPE] = "nteventlogtype";
 
 // Convert all fieldnames to lowercase to avoid problems with case sensitive array keys later 
 foreach( $dbmapping as &$myMapping ) 

src/lang/en/main.php

@@ -229,6 +229,8 @@ $content['LN_GEN_AUTH_LDAP_OPTIONS'] = "LDAP Authentication Options";
 $content['LN_DETAILS_FORSYSLOGMSG'] = "Details for the syslog messages with id";
 $content['LN_DETAILS_DETAILSFORMSG'] = "Details for message id";
 $content['LN_DETAIL_BACKTOLIST'] = "Back to Listview";
+$content['LN_DETAIL_DYNAMIC_FIELDS'] = "Dynamic fields";
+
 
 // Login Site
 $content['LN_LOGIN_DESCRIPTION'] = "Use this form to login into LogAnalyzer. ";

src/templates/details.html

@@ -76,13 +76,36 @@
 			<B>{FieldCaption}</B>
 		</td>
 		<td width="100%" align="{FieldAlign}" class="{cssclass}" {fieldbgcolor} valign="top">
-		<B>
-		</B>
 		{fieldvalue}
 		</td>
 	</tr>
 	<!-- ENDIF fieldenabled="true" -->
 	<!-- END fields -->
+	<!-- IF dynamicfieldsenabled="true" -->
+	<tr>
+		<td width="200" class="cellmenu1_naked" align="left" nowrap>
+			<B>{LN_DETAIL_DYNAMIC_FIELDS}</B>
+		</td>
+		<td width="100%" align="center" class="line1" valign="top">
+		<br/>
+
+		<table border="0" cellpadding="2" cellspacing="1" bgcolor="#DDDDDD" width="600" class="with_border_alternative">
+		<!-- BEGIN dynamicfields -->
+		<tr>
+			<td align="left" class="cellmenu2">
+			{dynfieldkey}
+			</td>
+			<td align="left" class="{dyncssclass}">
+				{dynfieldvalue}
+			</td>
+		</tr>
+		<!-- END dynamicfields -->
+		</table>
+
+		<br/>
+		</td>
+	</tr>
+	<!-- ENDIF dynamicfieldsenabled="true" -->
 	</table>
 	<!-- ENDIF messageenabled="true" -->