From 7afe7b52cd3c99a9b306068904cef3875bbafff5 Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Mon, 17 Nov 2008 14:20:40 +0100 Subject: [PATCH 1/4] Fixed typo of new column "DBRecordsPerQuery" in admin/sources.php causing adding new sources to fail --- src/admin/sources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/sources.php b/src/admin/sources.php index 56d5d90..032a226 100644 --- a/src/admin/sources.php +++ b/src/admin/sources.php @@ -706,7 +706,7 @@ if ( isset($_POST['op']) ) } else if ( $content['SourceType'] == SOURCE_DB || $content['SourceType'] == SOURCE_PDO ) { - $sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, SourceDBRecordsPerQuery, userid, groupid) + $sqlquery = "INSERT INTO " . DB_SOURCES . " (Name, Description, SourceType, MsgParserList, MsgNormalize, MsgSkipUnparseable, ViewID, DBTableType, DBType, DBServer, DBName, DBUser, DBPassword, DBTableName, DBEnableRowCounting, DBRecordsPerQuery, userid, groupid) VALUES ('" . $content['Name'] . "', '" . $content['Description'] . "', " . $content['SourceType'] . ", From 5ddc6f1374ae5a861fbaecdb44094da28366e106 Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Tue, 18 Nov 2008 17:05:00 +0100 Subject: [PATCH 2/4] Added support for logstream maintenance by using php over commandline --- src/classes/logstreamdb.class.php | 4 +- src/cron/maintenance.bat | 2 + src/cron/maintenance.php | 224 ++++++++++++++++++++++++++++++ src/cron/maintenance.sh | 4 + src/include/functions_common.php | 92 ++++++++---- src/lang/en/admin.php | 13 ++ 6 files changed, 307 insertions(+), 32 deletions(-) create mode 100644 src/cron/maintenance.bat create mode 100644 src/cron/maintenance.php create mode 100644 src/cron/maintenance.sh diff --git a/src/classes/logstreamdb.class.php b/src/classes/logstreamdb.class.php index c32cc90..59a4e85 100644 --- a/src/classes/logstreamdb.class.php +++ b/src/classes/logstreamdb.class.php @@ -664,8 +664,8 @@ class LogStreamDB extends LogStream { // Get affected rows and return! $rowcount = mysql_affected_rows(); - // Free query now - mysql_free_result ($myQuery); + // Free result not needed here! + //mysql_free_result ($myQuery); } } diff --git a/src/cron/maintenance.bat b/src/cron/maintenance.bat new file mode 100644 index 0000000..dfd820d --- /dev/null +++ b/src/cron/maintenance.bat @@ -0,0 +1,2 @@ + +C:\php\php.exe -q -a .\maintenance.php cleardata 2 olderthan 86400 diff --git a/src/cron/maintenance.php b/src/cron/maintenance.php new file mode 100644 index 0000000..44b0287 --- /dev/null +++ b/src/cron/maintenance.php @@ -0,0 +1,224 @@ + This file is meant for command line maintenance + * + * All directives are explained within this file + * + * Copyright (C) 2008 Adiscon GmbH. + * + * This file is part of phpLogCon. + * + * PhpLogCon is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * PhpLogCon is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with phpLogCon. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this + * distribution + ********************************************************************* +*/ + +// *** Default includes and procedures *** // +define('IN_PHPLOGCON', true); +$gl_root_path = './../'; + +// Now include necessary include files! +include($gl_root_path . 'include/functions_common.php'); + +// Set commandline mode for the script +define('IN_PHPLOGCON_COMMANDLINE', true); +$content['IN_PHPLOGCON_COMMANDLINE'] = true; +InitPhpLogCon(); +InitSourceConfigs(); + +// Init admin langauge file now! +IncludeLanguageFile( $gl_root_path . '/lang/' . $LANG . '/admin.php' ); +// *** *** // + +// --- Helper functions +/* +* Cleans data in desired logstream +*/ +function CleanData($optParam1, $optParam2, $optParam3, $optParam4) +{ + global $content, $gl_root_path; + + // Get Source reference! + $mySource = $content['Sources'][ $content['SOURCEID'] ]; + + // Check Source Type + if ( $mySource['SourceType'] == SOURCE_DB || $mySource['SourceType'] == SOURCE_PDO ) + { + // Include LogStream facility + include($gl_root_path . 'classes/logstream.class.php'); + + //Debug Output + PrintHTMLDebugInfo( DEBUG_INFO, "CleanData", GetAndReplaceLangStr($content["LN_CMD_CLEANINGDATAFOR"], $mySource['Name']) ); + + // Create LogStream Object + $stream = $mySource['ObjRef']->LogStreamFactory($mySource['ObjRef']); + $res = $stream->Verify(); + if ( $res == SUCCESS ) + { + // Gather Database Stats + $content['ROWCOUNT'] = $stream->GetLogStreamTotalRowCount(); + if ( isset($content['ROWCOUNT']) ) + { + //Debug Output + PrintHTMLDebugInfo( DEBUG_INFO, "CleanData", GetAndReplaceLangStr($content["LN_CMD_ROWSFOUND"], $content['ROWCOUNT'], $mySource['Name']) ); + + if ( $optParam1 != NULL ) + { + if ( $optParam1 == "all" ) + { + $timestamp = 0; + } + else if ( $optParam1 == "olderthan" && $optParam2 != NULL ) + { + // Take current time and subtract Seconds + $nSecondsSubtract = intval($optParam2); + $timestamp = time() - $nSecondsSubtract; + } + else if ( $optParam1 == "date" && $optParam2 != NULL && $optParam3 != NULL && $optParam4 != NULL ) + { + // Generate Timestamp + $timestamp = mktime( 0, 0, 0, intval($optParam2), intval($optParam3), intval($optParam4) ); + } + else + // Print error and die! + DieWithErrorMsg( $content["LN_CMD_WRONGSUBOPORMISSING"] ); + + // Continue with delete only if $timestamp is set! + if ( isset($timestamp) ) + { + //Debug Output + PrintHTMLDebugInfo( DEBUG_INFO, "CleanData", GetAndReplaceLangStr($content["LN_CMD_DELETINGOLDERTHEN"], date("Y-m-d", $timestamp) ) ); + + // Now perform the data cleanup! + $content['affectedrows'] = $stream->CleanupLogdataByDate($timestamp); + + if ( isset($content['affectedrows']) ) + { + //Debug Output + PrintHTMLDebugInfo( DEBUG_INFO, "CleanData", GetAndReplaceLangStr($content["LN_CMD_DELETEDROWS"], $content['affectedrows']) ); + } + else + // Print error and die! + DieWithErrorMsg( GetAndReplaceLangStr($content["LN_CMD_FAILEDTOCLEANDATA"], $mySource['Name']) ); + } + } + else + // Print error and die! + DieWithErrorMsg( $content["LN_CMD_SUBPARAM1MISSING"] ); + } + else + // Print error and die! + DieWithErrorMsg( GetAndReplaceLangStr($content["LN_CMD_COULDNOTGETROWCOUNT"], $mySource['Name']) ); + } + else + { + // Print error and die! + $szErroMsg = GetAndReplaceLangStr($content["LN_SOURCES_ERROR_WITHINSOURCE"],$mySource['Name'], GetErrorMessage($res)); + if ( isset($extraErrorDescription) ) + $szErroMsg .= "\r\n\r\n" . GetAndReplaceLangStr( $content['LN_SOURCES_ERROR_EXTRAMSG'], $extraErrorDescription); + DieWithErrorMsg( $szErroMsg ); + } + } + else + // Print error and die! + DieWithErrorMsg( GetAndReplaceLangStr($content["LN_SOURCES_ERROR_NOCLEARSUPPORT"], $content['SOURCEID']) ); +} +// --- + + +// --- BEGIN Custom Code + //Additional Includes + include($gl_root_path . 'include/functions_debugoutput.php'); + + // Run into Commandline part now! + /* Only run if we are in command line mode + * + * Possible Operation Types: + * cleandata = If you want to clear data from a logstream source, you can use the operation type. + * Be carefull using this option, any deletion process cannot be undone! + * Sample 1: Delete all data in the logstream with id 2 + * php maintenance.php cleardata 2 all + * Sample 2: Delete all data older then 60 seconds in the logstream with id 2 + * php maintenance.php cleardata 2 olderthan 60 + * Sample 3: Delete all data before 2008-11-18 in the logstream with id 2 + * php maintenance.php cleardata 2 date 11 18 2008 + * + */ + + // Init DebugHeader + PrintDebugInfoHeader(); + + // --- Now read command line args! + // Operation argv + if ( isset($_SERVER["argv"][1]) ) + $operation = $_SERVER["argv"][1]; + else + DieWithErrorMsg( $content["LN_CMD_NOOP"] ); + + // SourceID argv + if ( isset($_SERVER["argv"][2]) ) + { + // Set to SourceID property! + $content['SOURCEID'] = intval( $_SERVER["argv"][2] ); + + // Check if exists + if ( !isset($content['Sources'][ $content['SOURCEID'] ]) ) + DieWithErrorMsg( GetAndReplaceLangStr($content["LN_CMD_LOGSTREAMNOTFOUND"], $content['SOURCEID']) ); + } + else + DieWithErrorMsg( $content["LN_CMD_NOLOGSTREAM"] ); + + + // First Optional Parameter + if ( isset($_SERVER["argv"][3]) ) + $optparam1 = $_SERVER["argv"][3]; + else + $optparam1 = NULL; + + // Second Optional Parameter + if ( isset($_SERVER["argv"][4]) ) + $optparam2 = $_SERVER["argv"][4]; + else + $optparam2 = NULL; + + // Third Optional Parameter + if ( isset($_SERVER["argv"][5]) ) + $optParam3 = $_SERVER["argv"][5]; + else + $optParam3 = NULL; + + // fourth Optional Parameter + if ( isset($_SERVER["argv"][6]) ) + $optParam4 = $_SERVER["argv"][6]; + else + $optParam4 = NULL; + // --- + + // --- Operation Handling now + if ( $operation == "cleandata" ) + { + // Run Parser only + CleanData($optparam1, $optparam2, $optParam3, $optParam4); + } + // --- +// --- + +?> \ No newline at end of file diff --git a/src/cron/maintenance.sh b/src/cron/maintenance.sh new file mode 100644 index 0000000..d068067 --- /dev/null +++ b/src/cron/maintenance.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd /var/www/phplogcon/cron/ +php ./maintenance.php cleardata 2 olderthan 86400 diff --git a/src/include/functions_common.php b/src/include/functions_common.php index d3e9750..f4a444d 100644 --- a/src/include/functions_common.php +++ b/src/include/functions_common.php @@ -484,10 +484,15 @@ function InitPhpDebugMode() function CheckAndSetRunMode() { global $content, $RUNMODE; + // Set to command line mode if argv is set! - if ( !isset($_SERVER["GATEWAY_INTERFACE"]) ) + if ( !isset($_SERVER["SERVER_SOFTWARE"]) ) $RUNMODE = RUNMODE_COMMANDLINE; + // Check if we require the command line mode! + if ( defined("IN_PHPLOGCON_COMMANDLINE") && $RUNMODE != RUNMODE_COMMANDLINE ) + DieWithErrorMsg( "This PHP Script cannot be run within the webserver process, it designed to run over command line." ); + // Obtain max_execution_time $content['MaxExecutionTime'] = ini_get("max_execution_time"); @@ -722,7 +727,10 @@ function InitConfigurationValues() if ( GetConfigSetting("UserDBLoginRequired", false) ) { // Redirect USER if not on loginpage or installpage! - if ( !defined("IS_NOLOGINPAGE") && !defined("IN_PHPLOGCON_INSTALL") ) + if ( !defined("IS_NOLOGINPAGE") && + !defined("IN_PHPLOGCON_INSTALL") && + !defined("IN_PHPLOGCON_COMMANDLINE") + ) RedirectToUserLogin(); } else if ( defined('IS_ADMINPAGE') ) @@ -898,37 +906,62 @@ function CheckUrlOrIP($ip) function DieWithErrorMsg( $szerrmsg ) { - global $gl_root_path, $content; - echo - "phpLogCon :: Critical Error occured" . - "

" . - "". - "" . - "" . - "
" . - "

Critical Error occured

" . - "
Errordetails:
" . - $szerrmsg . - "

" . - ""; + global $RUNMODE, $content, $gl_root_path; + if ( $RUNMODE == RUNMODE_COMMANDLINE ) + { + print("\n\n\tCritical Error occured\t-\tErrordetails:\n"); + print("\t" . $szerrmsg . "\n\n"); + print("\tTerminating now!\n"); + } + else if ( $RUNMODE == RUNMODE_WEBSERVER ) + { + print( + "phpLogCon :: Critical Error occured" . + "

" . + "". + "" . + "" . + "
" . + "

Critical Error occured

" . + "
Errordetails:
" . + $szerrmsg . + "

" . + "" + ); + } + + // Abort further execution exit; } function DieWithFriendlyErrorMsg( $szerrmsg ) { - global $gl_root_path, $content; - echo - "phpLogCon :: Error occured" . - "

" . - "". - "" . - "" . - "
" . - "

Error occured

" . - "
Errordetails:
" . - $szerrmsg . - "

" . - ""; + global $RUNMODE, $content, $gl_root_path; + if ( $RUNMODE == RUNMODE_COMMANDLINE ) + { + print("\n\n\t\tError occured\n"); + print("\t\tErrordetails:\t" . $szerrmsg . "\n"); + print("\t\tTerminating now!\n"); + } + else if ( $RUNMODE == RUNMODE_WEBSERVER ) + { + + print( + "phpLogCon :: Error occured" . + "

" . + "". + "" . + "" . + "
" . + "

Error occured

" . + "
Errordetails:
" . + $szerrmsg . + "

" . + "" + ); + } + + // Abort further execution exit; } @@ -1316,8 +1349,7 @@ function StartPHPSession() if ( $RUNMODE == RUNMODE_WEBSERVER ) { // This will start the session - if (session_id() == "") - session_start(); + @session_start(); if ( !isset($_SESSION['SESSION_STARTED']) ) $_SESSION['SESSION_STARTED'] = "true"; diff --git a/src/lang/en/admin.php b/src/lang/en/admin.php index 70699d4..919c82d 100644 --- a/src/lang/en/admin.php +++ b/src/lang/en/admin.php @@ -344,5 +344,18 @@ $content['LN_PARSERS_ERROR_HASBEENADDED'] = "All required settings ('%2' custom $content['LN_PARSERS_ERROR_NOFIELDS'] = "The Message Parser '%1' does not have any custom fields to add."; $content['LN_PARSERS_'] = ""; +// Command Line stuff +$content['LN_CMD_NOOP'] = "Operation parameter is missing"; +$content['LN_CMD_NOLOGSTREAM'] = "The logstream source parameter is missing"; +$content['LN_CMD_LOGSTREAMNOTFOUND'] = "Logstream Source with ID '%1' could not be found in the Database!"; +$content['LN_CMD_COULDNOTGETROWCOUNT'] = "Could not obtain rowcount from logstream source '%1'"; +$content['LN_CMD_SUBPARAM1MISSING'] = "Subparameter 1 is missing, it should be set to 'all', 'since' or 'date'. For more details see the documentation."; +$content['LN_CMD_WRONGSUBOPORMISSING'] = "Either the sub-operation is wrong, or another parameter is missing"; +$content['LN_CMD_FAILEDTOCLEANDATA'] = "Failed to cleandata for the logstream '%1'."; +$content['LN_CMD_CLEANINGDATAFOR'] = "Cleaning data for logstream source '%1'."; +$content['LN_CMD_ROWSFOUND'] = "Successfully connected and found '%1' rows in the logstream source."; +$content['LN_CMD_DELETINGOLDERTHEN'] = "Performing deletion of data entries older then '%1'."; +$content['LN_CMD_DELETEDROWS'] = "Successfully Deleted '%1' rows in the logstream source.'"; +$content['LN_CMD_'] = ""; ?> \ No newline at end of file From c6633465c57374e9115b4c19582949644f680102 Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Tue, 18 Nov 2008 17:06:44 +0100 Subject: [PATCH 3/4] Added missing translations --- src/lang/de/admin.php | 14 ++++++++++++++ src/lang/pt_BR/admin.php | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/lang/de/admin.php b/src/lang/de/admin.php index 16f4956..60cb72b 100644 --- a/src/lang/de/admin.php +++ b/src/lang/de/admin.php @@ -331,4 +331,18 @@ $content['LN_PARSERS_ERROR_HASBEENADDED'] = "All required settings ('%2' custom $content['LN_PARSERS_ERROR_NOFIELDS'] = "The Message Parser '%1' does not have any custom fields to add."; $content['LN_PARSERS_'] = ""; +// Command Line stuff +$content['LN_CMD_NOOP'] = "Operation parameter is missing"; +$content['LN_CMD_NOLOGSTREAM'] = "The logstream source parameter is missing"; +$content['LN_CMD_LOGSTREAMNOTFOUND'] = "Logstream Source with ID '%1' could not be found in the Database!"; +$content['LN_CMD_COULDNOTGETROWCOUNT'] = "Could not obtain rowcount from logstream source '%1'"; +$content['LN_CMD_SUBPARAM1MISSING'] = "Subparameter 1 is missing, it should be set to 'all', 'since' or 'date'. For more details see the documentation."; +$content['LN_CMD_WRONGSUBOPORMISSING'] = "Either the sub-operation is wrong, or another parameter is missing"; +$content['LN_CMD_FAILEDTOCLEANDATA'] = "Failed to cleandata for the logstream '%1'."; +$content['LN_CMD_CLEANINGDATAFOR'] = "Cleaning data for logstream source '%1'."; +$content['LN_CMD_ROWSFOUND'] = "Successfully connected and found '%1' rows in the logstream source."; +$content['LN_CMD_DELETINGOLDERTHEN'] = "Performing deletion of data entries older then '%1'."; +$content['LN_CMD_DELETEDROWS'] = "Successfully Deleted '%1' rows in the logstream source.'"; +$content['LN_CMD_'] = ""; + ?> \ No newline at end of file diff --git a/src/lang/pt_BR/admin.php b/src/lang/pt_BR/admin.php index 16f4956..60cb72b 100644 --- a/src/lang/pt_BR/admin.php +++ b/src/lang/pt_BR/admin.php @@ -331,4 +331,18 @@ $content['LN_PARSERS_ERROR_HASBEENADDED'] = "All required settings ('%2' custom $content['LN_PARSERS_ERROR_NOFIELDS'] = "The Message Parser '%1' does not have any custom fields to add."; $content['LN_PARSERS_'] = ""; +// Command Line stuff +$content['LN_CMD_NOOP'] = "Operation parameter is missing"; +$content['LN_CMD_NOLOGSTREAM'] = "The logstream source parameter is missing"; +$content['LN_CMD_LOGSTREAMNOTFOUND'] = "Logstream Source with ID '%1' could not be found in the Database!"; +$content['LN_CMD_COULDNOTGETROWCOUNT'] = "Could not obtain rowcount from logstream source '%1'"; +$content['LN_CMD_SUBPARAM1MISSING'] = "Subparameter 1 is missing, it should be set to 'all', 'since' or 'date'. For more details see the documentation."; +$content['LN_CMD_WRONGSUBOPORMISSING'] = "Either the sub-operation is wrong, or another parameter is missing"; +$content['LN_CMD_FAILEDTOCLEANDATA'] = "Failed to cleandata for the logstream '%1'."; +$content['LN_CMD_CLEANINGDATAFOR'] = "Cleaning data for logstream source '%1'."; +$content['LN_CMD_ROWSFOUND'] = "Successfully connected and found '%1' rows in the logstream source."; +$content['LN_CMD_DELETINGOLDERTHEN'] = "Performing deletion of data entries older then '%1'."; +$content['LN_CMD_DELETEDROWS'] = "Successfully Deleted '%1' rows in the logstream source.'"; +$content['LN_CMD_'] = ""; + ?> \ No newline at end of file From b2a1df9ca036dcb50f2aa37c4980ed6a95a21268 Mon Sep 17 00:00:00 2001 From: Andre Lorbach Date: Tue, 18 Nov 2008 17:10:11 +0100 Subject: [PATCH 4/4] Added changelog entry --- ChangeLog | 9 ++++++++- src/include/functions_common.php | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ed4e1d9..5ddd263 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,12 @@ --------------------------------------------------------------------------- -Version 2.5.17 (beta), 2008-11-03 +Version 2.5.19 (beta), 2008-11-18 +- Added a new "cron" folder which contains a maintenance.php script. + This script can be used on command line level for database maintenance. + For more information, see the documentation. +- Fixed typo of new column "DBRecordsPerQuery" in admin/sources.php + which caused an error when adding new database logstream sources. +--------------------------------------------------------------------------- +Version 2.5.18 (beta), 2008-11-12 - Added logstream statistic and maintenance option which are accessable in the sources admin panel. You can view overall stats of database logstreams, and cleanup data based on the date field. diff --git a/src/include/functions_common.php b/src/include/functions_common.php index f4a444d..d04806e 100644 --- a/src/include/functions_common.php +++ b/src/include/functions_common.php @@ -66,7 +66,7 @@ $LANG_EN = "en"; // Used for fallback $LANG = "en"; // Default language // Default Template vars -$content['BUILDNUMBER'] = "2.5.18"; +$content['BUILDNUMBER'] = "2.5.19"; $content['TITLE'] = "phpLogCon :: Release " . $content['BUILDNUMBER']; // Default page title $content['BASEPATH'] = $gl_root_path; $content['SHOW_DONATEBUTTON'] = true; // Default = true!