From b9e4957110ddef69c963bf6d4df471bf89fed616 Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Tue, 17 Nov 2009 17:07:15 +0100 Subject: [PATCH] Implemented missing new consolidation functions into logstreampdo class --- src/classes/logstreamdb.class.php | 98 ------ src/classes/logstreampdo.class.php | 328 ++++++++++++++---- .../reports/report.eventlog.monilog.class.php | 66 ++-- 3 files changed, 286 insertions(+), 206 deletions(-) diff --git a/src/classes/logstreamdb.class.php b/src/classes/logstreamdb.class.php index 4e4d0d1..d54dcf4 100644 --- a/src/classes/logstreamdb.class.php +++ b/src/classes/logstreamdb.class.php @@ -412,36 +412,6 @@ class LogStreamDB extends LogStream { // Increment for the Footer Stats $querycount++; } -//echo $szSql . "
" . $this->_lastPageUID; -//exit; - -/* OLD CODE - // Obtain last UID of renough records are available! - if ( $this->_totalRecordCount > $this->_logStreamConfigObj->_pageCount ) - { - // Get SQL Statement without properties - $szSql = $this->CreateSQLStatement(-1, false); - - $limitbegin = $this->_totalRecordCount - $this->_logStreamConfigObj->_pageCount; - - // Append LIMIT clause - $szSql .= " LIMIT " . $limitbegin . ", 1"; - - // Perform Database Query - if ($myQuery = mysql_query($szSql, $this->_dbhandle)) - { - // obtain first and only row - $myRow = mysql_fetch_row($myQuery); - $this->_lastPageUID = $myRow[0]; - - // Free query now - mysql_free_result ($myQuery); - } - - // Increment for the Footer Stats - $querycount++; - } -*/ // Return result! return $this->_lastPageUID; @@ -533,7 +503,6 @@ class LogStreamDB extends LogStream { return null; } - /** * Implementation of GetLogStreamTotalRowCount * @@ -571,7 +540,6 @@ class LogStreamDB extends LogStream { return $rowcount; } - /** * Implementation of the CleanupLogdataByDate function! Returns affected rows! */ @@ -614,7 +582,6 @@ class LogStreamDB extends LogStream { return $rowcount; } - /** * Implementation of ConsolidateItemListByField * @@ -777,8 +744,6 @@ class LogStreamDB extends LogStream { else // Only Include ConsolidateField $myDBQueryFields = $myDBConsFieldName . ", "; - - if ( $szConsFieldId == $szSortFieldId ) $myDBSortedFieldName = "ItemCount"; else @@ -837,21 +802,11 @@ class LogStreamDB extends LogStream { $aNewRow[$szConsFieldId] = $myFieldValue; else $aNewRow[$myFieldName] = $myFieldValue; - -// if ( in_array($myFieldName, $this->_arrProperties) ) -// echo "!!"; -//print_r ( $this->_arrProperties ); -// echo $myFieldName; -// exit; } // Add new row to result $aResult[] = $aNewRow; - -// $aResult[ $myRow[$myDBGroupByFieldName] ] = $myRow; } -// $aResult[] = $myRow; -// $aResult[ $myRow[$mySelectFieldName] ] = $myRow['TotalCount']; // return finished array if ( count($aResult) > 0 ) @@ -1154,57 +1109,6 @@ class LogStreamDB extends LogStream { return SUCCESS; } - /* - * This function only reads the uID values from the database. Using this method, - * it will be much faster to find the starting uID point we need when paging is used. - */ -/* OBSELETE CODE - private function ReadNextIDsFromDB() - { - global $querycount; - - // Get SQL Statement without properties - $szSql = $this->CreateSQLStatement(-1, false); - - // Append LIMIT clause - $szSql .= " LIMIT " . $this->_currentRecordStart . ", " . $this->_logStreamConfigObj->IDsPerQuery; - - // Perform Database Query - $myquery = mysql_query($szSql, $this->_dbhandle); - if ( !$myquery ) - { - $this->PrintDebugError("Invalid SQL: ".$szSql); - return ERROR_DB_QUERYFAILED; - } - - // Copy rows into the buffer! - $iBegin = $this->_currentRecordNum; - while ($myRow = mysql_fetch_array($myquery, MYSQL_ASSOC)) - { - $this->bufferedRecords[$iBegin] = $myRow; - $iBegin++; - } - - // Free Query ressources - mysql_free_result ($myquery); - - // Only obtain count if enabled and not done before - if ( $this->_logStreamConfigObj->DBEnableRowCounting && $this->_totalRecordCount == -1 ) - { - $this->_totalRecordCount = $this->GetRowCountFromTable(); - - if ( $this->_totalRecordCount <= 0 ) - return ERROR_NOMORERECORDS; - } - - // Increment for the Footer Stats - $querycount++; - - // return success state if reached this point! - return SUCCESS; - } -*/ - /* * Destroy the SQL QUery! */ @@ -1313,8 +1217,6 @@ class LogStreamDB extends LogStream { // Output Debug Informations OutputDebugMessage("LogStreamDB|CreateMainSQLQuery: Created SQL Query:
" . $szSql, DEBUG_DEBUG); -//print ("LogStreamDB|CreateMainSQLQuery: Created SQL Query:
" . $szSql); - // return success state if reached this point! return SUCCESS; } diff --git a/src/classes/logstreampdo.class.php b/src/classes/logstreampdo.class.php index d4e81bf..9548524 100644 --- a/src/classes/logstreampdo.class.php +++ b/src/classes/logstreampdo.class.php @@ -681,6 +681,142 @@ class LogStreamPDO extends LogStream { } + /** + * Implementation of ConsolidateItemListByField + * + * In the native MYSQL Logstream, the database will do most of the work + * + * @return integer Error stat + */ + public function ConsolidateItemListByField($szConsFieldId, $nRecordLimit, $szSortFieldId, $nSortingOrder) + { + global $content, $dbmapping, $fields; + + // Copy helper variables, this is just for better readability + $szTableType = $this->_logStreamConfigObj->DBTableType; + + // Check if fields are available + if ( !isset($dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId]) || !isset($dbmapping[$szTableType]['DBMAPPINGS'][$szSortFieldId]) ) + return ERROR_DB_DBFIELDNOTFOUND; + + // --- Set Options + $nConsFieldType = $fields[$szConsFieldId]['FieldType']; + + if ( $nSortingOrder == SORTING_ORDER_DESC ) + $szSortingOrder = "DESC"; + else + $szSortingOrder = "ASC"; + // --- + + // --- Set DB Field names + $myDBConsFieldName = $dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId]; + $myDBGroupByFieldName = $myDBConsFieldName; + $myDBQueryFields = $myDBConsFieldName . ", "; + + // Set Sorted Field + if ( $szConsFieldId == $szSortFieldId ) + $myDBSortedFieldName = "ItemCount"; + else + $myDBSortedFieldName = $szSortFieldId; + // --- + + // Special handling for date fields + if ( $nConsFieldType == FILTER_TYPE_DATE ) + { + if ( $this->_logStreamConfigObj->DBType == DB_MYSQL || + $this->_logStreamConfigObj->DBType == DB_PGSQL ) + { + // Helper variable for the select statement + $mySelectFieldName = $myDBGroupByFieldName . "Grouped"; + $myDBQueryFieldName = "DATE( " . $myDBConsFieldName . ") AS " . $myDBGroupByFieldName ; + } + else if($this->_logStreamConfigObj->DBType == DB_MSSQL ) + { + // TODO FIND A WAY FOR MSSQL! + } + } + + // Set Limit String + if ( $nRecordLimit > 0 ) + { + // Append LIMIT in this case! + if ( $this->_logStreamConfigObj->DBType == DB_MYSQL || + $this->_logStreamConfigObj->DBType == DB_PGSQL ) + $szLimitSql = " LIMIT " . $nRecordLimit; + else + $szLimitSql = ""; + // TODO FIND A WAY FOR MSSQL! + } + else + $szLimitSql = ""; + + // Create SQL Where Clause! + if ( $this->_SQLwhereClause == "" ) + { + $res = $this->CreateSQLWhereClause(); + if ( $res != SUCCESS ) + return $res; + } + + // Create SQL String now! + $szSql = "SELECT " . + $myDBQueryFields . + "count(" . $myDBConsFieldName . ") as ItemCount " . + " FROM " . $this->_logStreamConfigObj->DBTableName . + $this->_SQLwhereClause . + " GROUP BY " . $myDBGroupByFieldName . + " ORDER BY " . $myDBSortedFieldName . " " . $szSortingOrder . + $szLimitSql ; + + // Perform Database Query + $this->_myDBQuery = $this->_dbhandle->query($szSql); + if ( !$this->_myDBQuery ) + return ERROR_DB_QUERYFAILED; + + if ( $this->_myDBQuery->rowCount() == 0 ) + { + $this->_myDBQuery = null; + return ERROR_NOMORERECORDS; + } + + // Initialize Array variable + $aResult = array(); + + // Init Helper counter + $iCount = 0; + + // read data records + while ( ($myRow = $this->_myDBQuery->fetch(PDO::FETCH_ASSOC)) && $iCount < $nRecordLimit) + { + // Create new row + $aNewRow = array(); + + foreach ( $myRow as $myFieldName => $myFieldValue ) + { + if ( $myFieldName == $dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId] ) + $aNewRow[$szConsFieldId] = $myFieldValue; + else + $aNewRow[$myFieldName] = $myFieldValue; + } + + // Increment Counter + $iCount++; + + // Add new row to result + $aResult[] = $aNewRow; + } + + // Delete handle + $this->_myDBQuery = null; + + // return finished array + if ( count($aResult) > 0 ) + return $aResult; + else + return ERROR_NOMORERECORDS; + } + + /** * Implementation of ConsolidateDataByField * @@ -688,91 +824,147 @@ class LogStreamPDO extends LogStream { * * @return integer Error stat */ - public function ConsolidateDataByField($szConsFieldId, $nRecordLimit, $szSortFieldId, $nSortingOrder, $bIncludeLogStreamFields = false) + public function ConsolidateDataByField($szConsFieldId, $nRecordLimit, $szSortFieldId, $nSortingOrder, $aIncludeCustomFields = null, $bIncludeLogStreamFields = false) { - global $content, $dbmapping; + global $content, $dbmapping, $fields;; // Copy helper variables, this is just for better readability $szTableType = $this->_logStreamConfigObj->DBTableType; - if ( isset($dbmapping[$szTableType]['DBMAPPINGS'][$szFieldId]) ) + // Check if fields are available + if ( !isset($dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId]) || !isset($dbmapping[$szTableType]['DBMAPPINGS'][$szSortFieldId]) ) + return ERROR_DB_DBFIELDNOTFOUND; + + // --- Set Options + $nConsFieldType = $fields[$szConsFieldId]['FieldType']; + + if ( $nSortingOrder == SORTING_ORDER_DESC ) + $szSortingOrder = "DESC"; + else + $szSortingOrder = "ASC"; + // --- + + // --- Set DB Field names + $myDBConsFieldName = $dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId]; + $myDBGroupByFieldName = $myDBConsFieldName; + + // Check which fields to include + if ( $aIncludeCustomFields != null ) { - // Get FieldType - $nFieldType = $fields[$szFieldId]['FieldType']; - - // Set DB Field name first! - $myDBFieldName = $dbmapping[$szTableType]['DBMAPPINGS'][$szFieldId]; - $myDBQueryFieldName = $myDBFieldName; - $mySelectFieldName = $myDBFieldName; - - // Special handling for date fields - if ( $nFieldType == FILTER_TYPE_DATE ) + $myDBQueryFields = ""; + foreach ( $aIncludeCustomFields as $myFieldName ) { - if ( $this->_logStreamConfigObj->DBType == DB_MYSQL || - $this->_logStreamConfigObj->DBType == DB_PGSQL ) - { - // Helper variable for the select statement - $mySelectFieldName = $mySelectFieldName . "grouped"; - $myDBQueryFieldName = "DATE( " . $myDBFieldName . ") AS " . $mySelectFieldName ; - } - else if($this->_logStreamConfigObj->DBType == DB_MSSQL ) - { - // TODO FIND A WAY FOR MSSQL! - } + if ( isset($dbmapping[$szTableType]['DBMAPPINGS'][$myFieldName]) ) + $myDBQueryFields .= $dbmapping[$szTableType]['DBMAPPINGS'][$myFieldName] . ", "; } + + // Append Sortingfield + if ( !in_array($szConsFieldId, $aIncludeCustomFields) ) + $myDBQueryFields .= $myDBConsFieldName . ", "; + } + else if ( $bIncludeLogStreamFields ) + { + $myDBQueryFields = ""; + foreach ( $this->_arrProperties as $myFieldName ) + { + if ( isset($dbmapping[$szTableType]['DBMAPPINGS'][$myFieldName]) ) + $myDBQueryFields .= $dbmapping[$szTableType]['DBMAPPINGS'][$myFieldName] . ", "; + } + } + else // Only Include ConsolidateField + $myDBQueryFields = $myDBConsFieldName . ", "; - // Create SQL String now! - $szSql = "SELECT " . - $myDBQueryFieldName . ", " . - "count(" . $myDBFieldName . ") as totalcount " . - " FROM " . $this->_logStreamConfigObj->DBTableName . - " GROUP BY " . $mySelectFieldName . - " ORDER BY totalcount DESC"; + if ( $szConsFieldId == $szSortFieldId ) + $myDBSortedFieldName = "ItemCount"; + else + $myDBSortedFieldName = $szSortFieldId; + // --- + + // Special handling for date fields + if ( $nConsFieldType == FILTER_TYPE_DATE ) + { + if ( $this->_logStreamConfigObj->DBType == DB_MYSQL || + $this->_logStreamConfigObj->DBType == DB_PGSQL ) + { + // Helper variable for the select statement + $mySelectFieldName = $myDBGroupByFieldName . "Grouped"; + $myDBQueryFieldName = "DATE( " . $myDBConsFieldName . ") AS " . $myDBGroupByFieldName ; + } + else if($this->_logStreamConfigObj->DBType == DB_MSSQL ) + { + // TODO FIND A WAY FOR MSSQL! + } + } + + // Set Limit String + if ( $nRecordLimit > 0 ) + { // Append LIMIT in this case! if ( $this->_logStreamConfigObj->DBType == DB_MYSQL || $this->_logStreamConfigObj->DBType == DB_PGSQL ) - $szSql .= " LIMIT " . $nRecordLimit; - - // Perform Database Query - $this->_myDBQuery = $this->_dbhandle->query($szSql); - if ( !$this->_myDBQuery ) - return ERROR_DB_QUERYFAILED; - - if ( $this->_myDBQuery->rowCount() == 0 ) - { - $this->_myDBQuery = null; - return ERROR_NOMORERECORDS; - } - - // Initialize Array variable - $aResult = array(); - - // read data records - $iCount = 0; - while ( ($myRow = $this->_myDBQuery->fetch(PDO::FETCH_ASSOC)) && $iCount < $nRecordLimit) - { - if ( isset($myRow[$mySelectFieldName]) ) - { - $aResult[] = $myRow; -// $aResult[ $myRow[$mySelectFieldName] ] = $myRow['totalcount']; - $iCount++; - } - } - - // Delete handle - $this->_myDBQuery = null; - - // return finished array - if ( count($aResult) > 0 ) - return $aResult; + $szLimitSql = " LIMIT " . $nRecordLimit; else - return ERROR_NOMORERECORDS; + $szLimitSql = ""; + // TODO FIND A WAY FOR MSSQL! } else + $szLimitSql = ""; + + // Create SQL String now! + $szSql = "SELECT " . + $myDBQueryFields . + "count(" . $myDBConsFieldName . ") as ItemCount " . + " FROM " . $this->_logStreamConfigObj->DBTableName . + $this->_SQLwhereClause . + " GROUP BY " . $myDBGroupByFieldName . + " ORDER BY " . $myDBSortedFieldName . " " . $szSortingOrder . + $szLimitSql ; + + // Perform Database Query + $this->_myDBQuery = $this->_dbhandle->query($szSql); + if ( !$this->_myDBQuery ) + return ERROR_DB_QUERYFAILED; + + if ( $this->_myDBQuery->rowCount() == 0 ) { - // return error code, field mapping not found - return ERROR_DB_DBFIELDNOTFOUND; + $this->_myDBQuery = null; + return ERROR_NOMORERECORDS; } + + // Initialize Array variable + $aResult = array(); + + // read data records + $iCount = 0; + + while ( ($myRow = $this->_myDBQuery->fetch(PDO::FETCH_ASSOC)) && ($nRecordLimit == 0 || $iCount < $nRecordLimit) ) + { + // Create new row + $aNewRow = array(); + + foreach ( $myRow as $myFieldName => $myFieldValue ) + { + if ( $myFieldName == $dbmapping[$szTableType]['DBMAPPINGS'][$szConsFieldId] ) + $aNewRow[$szConsFieldId] = $myFieldValue; + else + $aNewRow[$myFieldName] = $myFieldValue; + } + + // Add new row to result + $aResult[] = $aNewRow; + + // Increment Counter + $iCount++; + } + + // Delete handle + $this->_myDBQuery = null; + + // return finished array + if ( count($aResult) > 0 ) + return $aResult; + else + return ERROR_NOMORERECORDS; } diff --git a/src/classes/reports/report.eventlog.monilog.class.php b/src/classes/reports/report.eventlog.monilog.class.php index fbe8b29..fd69c52 100644 --- a/src/classes/reports/report.eventlog.monilog.class.php +++ b/src/classes/reports/report.eventlog.monilog.class.php @@ -222,9 +222,6 @@ class Report_monilog extends Report { private function ConsolidateEventsPerHost( $arrHosts ) { global $content; - - // Set Filter string -// $this->_streamObj->SetFilter( $this->_filterString ); // Now open the stream for data processing $res = $this->_streamObj->Open( $this->_arrProperties, true ); @@ -288,7 +285,6 @@ class Report_monilog extends Report { $content["report_consdata"][ $logArray[SYSLOG_HOST] ]['cons_events'][ $strChecksum ]['FirstEvent_Date'] = $logArray[SYSLOG_DATE]; $content["report_consdata"][ $logArray[SYSLOG_HOST] ]['cons_events'][ $strChecksum ]['LastEvent_Date'] = $logArray[SYSLOG_DATE]; } - } // Get next data record @@ -319,10 +315,7 @@ class Report_monilog extends Report { $tmpMyEvent['LastEvent_Date_Formatted'] = GetFormatedDate( $tmpMyEvent['LastEvent_Date'] ); $tmpMyEvent['syslogseverity_text'] = $content['filter_severity_list'][ $tmpMyEvent['syslogseverity'] ]["DisplayName"]; } - - } - } else return $ret; @@ -331,42 +324,35 @@ class Report_monilog extends Report { // Work done! return SUCCESS; } -/* - private function ResetBuffer() { - $this->_bEOS = false; - $this->_buffer = false; - $this->_buffer_length = 0; - $this->_p_buffer = -1; - } -*/ } +// --- Static helper function! - /** - * Helper function for multisorting multidimensional arrays - */ - function MultiSortArrayByItemCountDesc( $arrayFirst, $arraySecond ) - { - // Do not sort in this case - if ($arrayFirst['ItemCount'] == $arraySecond['ItemCount']) - return 0; - - // Move up or down - return ($arrayFirst['ItemCount'] < $arraySecond['ItemCount']) ? 1 : -1; - } - - /** - * Helper function for multisorting multidimensional arrays - */ - function MultiSortArrayByItemCountAsc( $arrayFirst, $arraySecond ) - { - // Do not sort in this case - if ($arrayFirst['ItemCount'] == $arraySecond['ItemCount']) - return 0; - - // Move up or down - return ($arrayFirst['ItemCount'] < $arraySecond['ItemCount']) ? -1 : 1; - } +/** +* Helper function for multisorting multidimensional arrays +*/ +function MultiSortArrayByItemCountDesc( $arrayFirst, $arraySecond ) +{ + // Do not sort in this case + if ($arrayFirst['ItemCount'] == $arraySecond['ItemCount']) + return 0; + + // Move up or down + return ($arrayFirst['ItemCount'] < $arraySecond['ItemCount']) ? 1 : -1; +} +/** +* Helper function for multisorting multidimensional arrays +*/ +function MultiSortArrayByItemCountAsc( $arrayFirst, $arraySecond ) +{ + // Do not sort in this case + if ($arrayFirst['ItemCount'] == $arraySecond['ItemCount']) + return 0; + + // Move up or down + return ($arrayFirst['ItemCount'] < $arraySecond['ItemCount']) ? -1 : 1; +} +// --- ?> \ No newline at end of file