From 4daa93225d77d9a0b85e1032e205481188a4cb8a Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Mon, 27 Oct 2008 17:14:32 +0100 Subject: [PATCH] Added new setting for MYSQL logstream sources to control the amount of data per query. This value can be used to tweak MYSQL performance for your environment. For example when filtering for fields which are generated by a message parser, highering the value does improve the database performance. Default is 100. --- src/admin/sources.php | 16 ++++++++++++++-- src/classes/logstreamdb.class.php | 2 +- src/include/db_update_v7.txt | 1 + src/include/functions_config.php | 1 + src/lang/de/admin.php | 1 + src/lang/de/main.php | 1 + src/lang/en/admin.php | 1 + src/lang/en/main.php | 1 + src/lang/pt_BR/admin.php | 1 + src/lang/pt_BR/main.php | 1 + src/templates/admin/admin_sources.html | 19 +++++++++++++++---- 11 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/admin/sources.php b/src/admin/sources.php index 01bdcf3..a31853f 100644 --- a/src/admin/sources.php +++ b/src/admin/sources.php @@ -102,6 +102,7 @@ if ( isset($_GET['op']) ) $content['SourceDBEnableRowCounting'] = "false"; $content['SourceDBEnableRowCounting_true'] = ""; $content['SourceDBEnableRowCounting_false'] = "checked"; + $content['SourceDBRecordsPerQuery'] = "100"; // General stuff $content['userid'] = null; @@ -188,6 +189,8 @@ if ( isset($_GET['op']) ) $content['SourceDBEnableRowCounting_true'] = ""; $content['SourceDBEnableRowCounting_false'] = "checked"; } + $content['SourceDBRecordsPerQuery'] = $mysource['DBRecordsPerQuery']; + if ( $mysource['userid'] != null ) $content['CHECKED_ISUSERONLY'] = "checked"; @@ -302,6 +305,7 @@ if ( isset($_POST['op']) ) if ( isset($_POST['SourceDBServer']) ) { $content['SourceDBServer'] = DB_RemoveBadChars($_POST['SourceDBServer']); } if ( isset($_POST['SourceDBTableName']) ) { $content['SourceDBTableName'] = DB_RemoveBadChars($_POST['SourceDBTableName']); } if ( isset($_POST['SourceDBUser']) ) { $content['SourceDBUser'] = DB_RemoveBadChars($_POST['SourceDBUser']); } + if ( isset($_POST['SourceDBRecordsPerQuery']) ) { $content['SourceDBRecordsPerQuery'] = DB_RemoveBadChars($_POST['SourceDBRecordsPerQuery']); } if ( isset($_POST['SourceDBPassword']) ) { $content['SourceDBPassword'] = DB_RemoveBadChars($_POST['SourceDBPassword']); } else {$content['SourceDBPassword'] = ""; } if ( isset($_POST['SourceDBEnableRowCounting']) ) { $content['SourceDBEnableRowCounting'] = DB_RemoveBadChars($_POST['SourceDBEnableRowCounting']); } // Extra Check for this property @@ -417,6 +421,11 @@ if ( isset($_POST['op']) ) $content['ISERROR'] = true; $content['ERROR_MSG'] = GetAndReplaceLangStr( $content['LN_SOURCES_ERROR_MISSINGPARAM'], $content['LN_CFG_DBUSER'] ); } + else if ( !is_numeric($content['SourceDBRecordsPerQuery']) ) + { + $content['ISERROR'] = true; + $content['ERROR_MSG'] = GetAndReplaceLangStr( $content['LN_SOURCES_ERROR_INVALIDVALUE'], $content['LN_CFG_DBRECORDSPERQUERY'] ); + } } else { @@ -453,7 +462,8 @@ if ( isset($_POST['op']) ) $tmpSource['DBTableName'] = $content['SourceDBTableName']; $tmpSource['DBUser'] = $content['SourceDBUser']; $tmpSource['DBPassword'] = $content['SourceDBPassword']; - $tmpSource['DBEnableRowCounting'] = $content['SourceDBEnableRowCounting']; + $tmpSource['DBEnableRowCounting'] = $content['SourceDBEnableRowCounting']; + $tmpSource['DBRecordsPerQuery'] = $content['SourceDBRecordsPerQuery']; $tmpSource['userid'] = $content['userid']; $tmpSource['groupid'] = $content['groupid']; } @@ -500,7 +510,7 @@ if ( isset($_POST['op']) ) } else if ( $content['SourceType'] == SOURCE_DB || $content['SourceType'] == SOURCE_PDO ) { - $sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, userid, groupid) + $sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, SourceDBRecordsPerQuery, userid, groupid) VALUES ('" . $content['Name'] . "', '" . $content['Description'] . "', " . $content['SourceType'] . ", @@ -516,6 +526,7 @@ if ( isset($_POST['op']) ) '" . $content['SourceDBPassword'] . "', '" . $content['SourceDBTableName'] . "', " . $content['SourceDBEnableRowCounting'] . ", + " . $content['SourceDBRecordsPerQuery'] . ", " . $content['userid'] . ", " . $content['groupid'] . " )"; @@ -573,6 +584,7 @@ if ( isset($_POST['op']) ) DBPassword = '" . $content['SourceDBPassword'] . "', DBTableName = '" . $content['SourceDBTableName'] . "', DBEnableRowCounting = " . $content['SourceDBEnableRowCounting'] . ", + DBRecordsPerQuery = " . $content['SourceDBRecordsPerQuery'] . ", userid = " . $content['userid'] . ", groupid = " . $content['groupid'] . " WHERE ID = " . $content['SOURCEID']; diff --git a/src/classes/logstreamdb.class.php b/src/classes/logstreamdb.class.php index bb06c14..2c9bfe7 100644 --- a/src/classes/logstreamdb.class.php +++ b/src/classes/logstreamdb.class.php @@ -878,7 +878,7 @@ class LogStreamDB extends LogStream { // Get SQL Statement $szSql = $this->CreateSQLStatement($uID); - + // Append LIMIT clause $szSql .= " LIMIT " . $this->_currentRecordStart . ", " . $this->_logStreamConfigObj->RecordsPerQuery; //echo $szSql . "
"; diff --git a/src/include/db_update_v7.txt b/src/include/db_update_v7.txt index 5f52953..e9026a8 100644 --- a/src/include/db_update_v7.txt +++ b/src/include/db_update_v7.txt @@ -1,5 +1,6 @@ -- New Database Structure Updates ALTER TABLE `logcon_sources` ADD `MsgSkipUnparseable` BOOL NOT NULL DEFAULT '0' AFTER `MsgNormalize` ; +ALTER TABLE `logcon_sources` ADD `DBRecordsPerQuery` INT NOT NULL DEFAULT '100' AFTER `DBEnableRowCounting` ; -- Insert data diff --git a/src/include/functions_config.php b/src/include/functions_config.php index 1be09e1..6db76b8 100644 --- a/src/include/functions_config.php +++ b/src/include/functions_config.php @@ -159,6 +159,7 @@ function InitSource(&$mysource) if ( isset($mysource['DBUser']) ) { $mysource['ObjRef']->DBUser = $mysource['DBUser']; } if ( isset($mysource['DBPassword']) ) { $mysource['ObjRef']->DBPassword = $mysource['DBPassword']; } if ( isset($mysource['DBEnableRowCounting']) ) { $mysource['ObjRef']->DBEnableRowCounting = $mysource['DBEnableRowCounting']; } + if ( isset($mysource['DBRecordsPerQuery']) ) { $mysource['ObjRef']->RecordsPerQuery = $mysource['DBRecordsPerQuery']; } } else if ( $mysource['SourceType'] == SOURCE_PDO ) { diff --git a/src/lang/de/admin.php b/src/lang/de/admin.php index 4905f71..e153c17 100644 --- a/src/lang/de/admin.php +++ b/src/lang/de/admin.php @@ -223,6 +223,7 @@ $content['LN_SOURCES_WARNDELETESEARCH'] = "Are you sure that you want to delete $content['LN_SOURCES_ERROR_DELSOURCE'] = "Deleting of the Source with id '%1' failed!"; $content['LN_SOURCES_ERROR_HASBEENDEL'] = "The Source '%1' has been successfully deleted!"; $content['LN_SOURCES_DESCRIPTION'] = "Source Description (Optional)"; +$content['LN_SOURCES_ERROR_INVALIDVALUE'] = "Invalid value for the paramater '%1'."; // Database Upgrade $content['LN_DBUPGRADE_TITLE'] = "phpLogCon Database Update"; diff --git a/src/lang/de/main.php b/src/lang/de/main.php index 574597a..a466c69 100644 --- a/src/lang/de/main.php +++ b/src/lang/de/main.php @@ -193,6 +193,7 @@ $content['LN_CFG_FIRSTSYSLOGSOURCE'] = "Erste Syslog Quelle"; $content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)"; $content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers"; $content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)"; + $content['LN_CFG_DBRECORDSPERQUERY'] = "Recordcount for database queries"; // Details page $content['LN_DETAILS_FORSYSLOGMSG'] = "Details für syslog-Nachrichten mit der ID"; diff --git a/src/lang/en/admin.php b/src/lang/en/admin.php index ec035e7..da98e03 100644 --- a/src/lang/en/admin.php +++ b/src/lang/en/admin.php @@ -227,6 +227,7 @@ $content['LN_SOURCES_WARNDELETESEARCH'] = "Are you sure that you want to delete $content['LN_SOURCES_ERROR_DELSOURCE'] = "Deleting of the Source with id '%1' failed!"; $content['LN_SOURCES_ERROR_HASBEENDEL'] = "The Source '%1' has been successfully deleted!"; $content['LN_SOURCES_DESCRIPTION'] = "Source Description (Optional)"; +$content['LN_SOURCES_ERROR_INVALIDVALUE'] = "Invalid value for the paramater '%1'."; // Database Upgrade $content['LN_DBUPGRADE_TITLE'] = "phpLogCon Database Update"; diff --git a/src/lang/en/main.php b/src/lang/en/main.php index 87ac132..cc5ad57 100644 --- a/src/lang/en/main.php +++ b/src/lang/en/main.php @@ -195,6 +195,7 @@ $content['LN_CFG_DBUSERLOGINREQUIRED'] = "Require user to be logged in"; $content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)"; $content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers"; $content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)"; +$content['LN_CFG_DBRECORDSPERQUERY'] = "Recordcount for database queries"; // Details page $content['LN_DETAILS_FORSYSLOGMSG'] = "Details for the syslog messages with id"; diff --git a/src/lang/pt_BR/admin.php b/src/lang/pt_BR/admin.php index 4905f71..e153c17 100644 --- a/src/lang/pt_BR/admin.php +++ b/src/lang/pt_BR/admin.php @@ -223,6 +223,7 @@ $content['LN_SOURCES_WARNDELETESEARCH'] = "Are you sure that you want to delete $content['LN_SOURCES_ERROR_DELSOURCE'] = "Deleting of the Source with id '%1' failed!"; $content['LN_SOURCES_ERROR_HASBEENDEL'] = "The Source '%1' has been successfully deleted!"; $content['LN_SOURCES_DESCRIPTION'] = "Source Description (Optional)"; +$content['LN_SOURCES_ERROR_INVALIDVALUE'] = "Invalid value for the paramater '%1'."; // Database Upgrade $content['LN_DBUPGRADE_TITLE'] = "phpLogCon Database Update"; diff --git a/src/lang/pt_BR/main.php b/src/lang/pt_BR/main.php index ce47da1..5e7064e 100644 --- a/src/lang/pt_BR/main.php +++ b/src/lang/pt_BR/main.php @@ -197,6 +197,7 @@ $content['LN_CFG_VIEW'] = "Selecione visão"; $content['LN_CFG_MSGPARSERS'] = "Message Parsers (comma seperated)"; $content['LN_CFG_NORMALIZEMSG'] = "Normalize Message within Parsers"; $content['LN_CFG_SKIPUNPARSEABLE'] = "Skip unparseable messages (Only works if msgparsers are configured!)"; + $content['LN_CFG_DBRECORDSPERQUERY'] = "Recordcount for database queries"; // Details page $content['LN_DETAILS_FORSYSLOGMSG'] = "Detalhes para a mensagem com id"; diff --git a/src/templates/admin/admin_sources.html b/src/templates/admin/admin_sources.html index 145ab72..2e19195 100644 --- a/src/templates/admin/admin_sources.html +++ b/src/templates/admin/admin_sources.html @@ -17,14 +17,16 @@ showvisibility("HiddenDatabaseTypeOptions"); hidevisibility("HiddenDiskTypeOptions"); - hidevisibility("HiddenDBTypesOptions"); + showvisibility("HiddenMYSQLSourceOptions"); + hidevisibility("HiddenPDOSourceOptions"); } else if (myfield.value == 3) { showvisibility("HiddenDatabaseTypeOptions"); hidevisibility("HiddenDiskTypeOptions"); - showvisibility("HiddenDBTypesOptions"); + showvisibility("HiddenPDOSourceOptions"); + hidevisibility("HiddenMYSQLSourceOptions"); } } @@ -136,7 +138,7 @@ - {LN_CFG_SKIPUNPARSEABLE} + {LN_CFG_SKIPUNPARSEABLE} @@ -184,7 +186,7 @@ {LN_CFG_DATABASETYPEOPTIONS} -
+
@@ -233,6 +235,15 @@
{LN_CFG_DBSTORAGEENGINE}
+
+ + + + + +
{LN_CFG_DBRECORDSPERQUERY}
+
+