From a3c6807ffd3f96a7adf22c13df7d38bda047055e Mon Sep 17 00:00:00 2001 From: Darold Gilles Date: Mon, 27 May 2013 22:27:18 +0200 Subject: [PATCH] Add -b | --build_date command line option to limit rebuilt to the given date, the format of its value can be yyyy-mm-dd, yyyy-mm or yyyy. This should be used to prevent rebuilding all html file from all data file. --- README | 31 +++++++++++++---------- SquidAnalyzer.pm | 57 +++++++++++++++++++++++++++++++------------ doc/SquidAnalyzer.pod | 31 +++++++++++++---------- doc/squidanalyzer.3 | 30 +++++++++++++---------- squid-analyzer | 22 ++++++++++++++--- 5 files changed, 113 insertions(+), 58 deletions(-) diff --git a/README b/README index 9bb4197..654d130 100644 --- a/README +++ b/README @@ -131,22 +131,27 @@ USAGE SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl script. Here are authorized usage: - squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] + Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] - -c | --configfile filename : path to the SquidAnalyzer configuration file. - By default: /etc/squidanalyzer.conf - -d | --debug : show debug informations. - -h | --help : show this message and exit. - -l | --logfile filename : path to the Squid logfile to parse. - By default: /var/log/squid/access.log - -p | --preserve number : used to set the statistic obsolescence in - number of month. Older stats will be removed. - -r | --rebuild : use this option to rebuild all html and graphs - output from all data files. - -v | version : show version and exit. + -c | --configfile filename : path to the SquidAnalyzer configuration file. + By default: /etc/squidanalyzer.conf + -b | --build_date date : set the day to be rebuilt, format: yyyy-mm-dd, + yyyy-mm or yyyy. Used with -r or --rebuild. + -d | --debug : show debug informations. + -h | --help : show this message and exit. + -l | --logfile filename : path to the Squid logfile to parse. + By default: /var/log/squid/access.log + -p | --preserve number : used to set the statistic obsolescence in + number of month. Older stats will be removed. + -r | --rebuild : use this option to rebuild all html and graphs + output from all data files. + -v | version : show version and exit. There is special options like --rebuild that force SquidAnalyzer to - rebuild all HTML reports, useful after an new feature or a bug fix. + rebuild all HTML reports, useful after an new feature or a bug fix. If + you want to limit the rebuild to a single day, a single month or year, + you can use the --build_date option by specifying the date part to + rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy. The --preserve option should be used if you want to rotate your statistics and data. The value is the number of months to keep, older diff --git a/SquidAnalyzer.pm b/SquidAnalyzer.pm index 7d4b062..79f3263 100644 --- a/SquidAnalyzer.pm +++ b/SquidAnalyzer.pm @@ -16,7 +16,7 @@ use strict; # make things properly BEGIN { use Exporter(); use vars qw($VERSION $COPYRIGHT $AUTHOR @ISA @EXPORT $ZCAT_PROG $BZCAT_PROG $RM_PROG); - use POSIX; + use POSIX qw/ strftime /; use IO::File; # Set all internal variable @@ -127,19 +127,21 @@ my %Translate = ( 'Click_year_stat' => 'Click on year\'s statistics link for details', 'Mime_graph_hits_title' => 'Mime Type Hits Statistics on', 'Mime_graph_bytes_title' => 'Mime Type Bytes Statistiques on', + 'User' => 'User', + 'Count' => 'Count', ); sub new { - my ($class, $conf_file, $log_file, $debug) = @_; + my ($class, $conf_file, $log_file, $debug, $rebuild) = @_; # Construct the class my $self = {}; bless $self, $class; # Initialize all variables - $self->_init($conf_file, $log_file, $debug); + $self->_init($conf_file, $log_file, $debug, $rebuild); # Return the instance return($self); @@ -184,6 +186,7 @@ sub parseFile my $line_count = 0; my $line_processed_count = 0; my $line_stored_count = 0; + # Read and parse each line of the access log file while ($line = <$logfile>) { chomp($line); @@ -382,7 +385,7 @@ sub _clear_stats sub _init { - my ($self, $conf_file, $log_file, $debug) = @_; + my ($self, $conf_file, $log_file, $debug, $rebuild) = @_; # Prevent for a call without instance if (!ref($self)) { @@ -398,7 +401,7 @@ sub _init $conf_file = 'squidanalyzer.conf'; } } - my %options = &parse_config($conf_file, $log_file); + my %options = &parse_config($conf_file, $log_file, $rebuild); # Use squid log file given as command line parameter $options{LogFile} = $log_file if ($log_file); @@ -432,13 +435,13 @@ sub _init die "ERROR: 'Output' configuration option must be set.\n"; } if (! -d $self->{Output}) { - die "ERROR: 'Output' dorectory $self->{Output} doesn't exists.\n"; + die "ERROR: 'Output' directory $self->{Output} doesn't exists.\n"; } $self->{LogFile} = $options{LogFile} || '/var/log/squid/access.log'; if (!$self->{LogFile}) { die "ERROR: 'LogFile' configuration option must be set.\n"; } - if (! -e $self->{LogFile}) { + if (!$rebuild && ! -e $self->{LogFile}) { die "ERROR: 'LogFile' $self->{LogFile} doesn't exists.\n"; } $self->{OrderUser} = lc($options{OrderUser}) || 'bytes'; @@ -507,7 +510,7 @@ sub _init } # Get the last parsing date for incremental parsing - if (-e "$self->{Output}/SquidAnalyzer.current") { + if (!$rebuild && -e "$self->{Output}/SquidAnalyzer.current") { my $current = new IO::File; unless($current->open("$self->{Output}/SquidAnalyzer.current")) { print STDERR "ERROR: Can't read file $self->{Output}/SquidAnalyzer.current, $!\n" if (!$self->{QuietMode}); @@ -711,7 +714,7 @@ sub _save_stat my $path = join('/', $year, $month, $day); $path =~ s/[\/]+$//; - #### Load history + #### Load history if we are not rebuilding a particular day if ($type eq 'day') { foreach my $d ("01" .. "31") { $self->_read_stat($year, $month, $d, 'day'); @@ -1167,6 +1170,25 @@ sub _print_footer } +sub check_build_date +{ + my ($self, $year, $month, $day) = @_; + + return 0 if (!$self->{build_date}); + + my ($y, $m, $d) = split(/\-/, $self->{build_date}); + + return 1 if ($year ne $y); + if ($m) { + return 1 if ($month && ($month ne $m)); + if ($d) { + return 1 if ($day && ($day ne $d)); + } + } + + return 0; +} + sub buildHTML { my ($self, $outdir) = @_; @@ -1204,6 +1226,7 @@ sub buildHTML closedir DIR; foreach my $y (sort {$a <=> $b} @years) { next if (!$y); + next if ($self->check_build_date($y)); # Remove the full year repository if it is older that the last date to preserve if ($p_year && ($y < $p_year)) { print STDERR "Removing obsolete statistics for year $y\n" if (!$self->{QuietMode}); @@ -1216,6 +1239,7 @@ sub buildHTML closedir DIR; foreach my $m (sort {$a <=> $b} @months) { next if (!$m); + next if ($self->check_build_date($y, $m)); # Remove the full month repository if it is older that the last date to preserve if ($p_year && ("$y$m" < "$p_year$p_month")) { print STDERR "Removing obsolete statistics for month $y-$m\n" if (!$self->{QuietMode}); @@ -1227,6 +1251,7 @@ sub buildHTML my @days = grep { /^\d{2}$/ && -d "$outdir/$y/$m/$_"} readdir(DIR); closedir DIR; foreach my $d (sort {$a <=> $b} @days) { + next if ($self->check_build_date($y, $m, $d)); next if ("$y$m$d" < "$old_year$old_month$old_day"); print STDERR "Generating daily statistics for day $y-$m-$d\n" if (!$self->{QuietMode}); $self->gen_html_output($outdir, $y, $m, $d); @@ -2479,7 +2504,7 @@ sub _print_top_url_stat if (exists $url_stat{$u}{users}) { print $out qq{
$u
- +
UserCount
}; my $k = 1; foreach my $user (sort { $url_stat{$u}{users}{$b} <=> $url_stat{$u}{users}{$a} } keys %{$url_stat{$u}{users}}) { @@ -2711,7 +2736,7 @@ sub _print_top_domain_stat if (exists $domain_stat{$u}{users}) { print $out qq{
*.$u
-
$Translate{'User'}$Translate{'Count'}
+
UserCount
}; my $k = 1; foreach my $user (sort { $domain_stat{$u}{users}{$b} <=> $domain_stat{$u}{users}{$a} } keys %{$domain_stat{$u}{users}}) { @@ -2867,7 +2892,7 @@ sub _gen_summary sub parse_config { - my ($file, $log_file) = @_; + my ($file, $log_file, $rebuild) = @_; die "FATAL: no configuration file!\n" if (!-e $file); @@ -2886,9 +2911,11 @@ sub parse_config print STDERR "Error: you must give a valid output directory. See option: Output\n"; exit 0; } - if (!$log_file && (!exists $opt{LogFile} || !-f $opt{LogFile})) { - print STDERR "Error: you must give the path to the Squid log file. See option: LogFile\n"; - exit 0; + if (!$rebuild || $log_file) { + if (!$log_file && (!exists $opt{LogFile} || !-f $opt{LogFile})) { + print STDERR "Error: you must give the path to the Squid log file. See option: LogFile\n"; + exit 0; + } } if (exists $opt{DateFormat}) { if ( ($opt{DateFormat} !~ m#\%y#) || (($opt{DateFormat} !~ m#\%m#) && ($opt{DateFormat} !~ m#\%M#) )|| ($opt{DateFormat} !~ m#\%d#) ) { diff --git a/doc/SquidAnalyzer.pod b/doc/SquidAnalyzer.pod index a20f056..96961d4 100644 --- a/doc/SquidAnalyzer.pod +++ b/doc/SquidAnalyzer.pod @@ -136,22 +136,27 @@ match your network and file system configuration. SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl script. Here are authorized usage: - squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] + Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] + + -c | --configfile filename : path to the SquidAnalyzer configuration file. + By default: /etc/squidanalyzer.conf + -b | --build_date date : set the day to be rebuilt, format: yyyy-mm-dd, + yyyy-mm or yyyy. Used with -r or --rebuild. + -d | --debug : show debug informations. + -h | --help : show this message and exit. + -l | --logfile filename : path to the Squid logfile to parse. + By default: /var/log/squid/access.log + -p | --preserve number : used to set the statistic obsolescence in + number of month. Older stats will be removed. + -r | --rebuild : use this option to rebuild all html and graphs + output from all data files. + -v | version : show version and exit. - -c | --configfile filename : path to the SquidAnalyzer configuration file. - By default: /etc/squidanalyzer.conf - -d | --debug : show debug informations. - -h | --help : show this message and exit. - -l | --logfile filename : path to the Squid logfile to parse. - By default: /var/log/squid/access.log - -p | --preserve number : used to set the statistic obsolescence in - number of month. Older stats will be removed. - -r | --rebuild : use this option to rebuild all html and graphs - output from all data files. - -v | version : show version and exit. There is special options like --rebuild that force SquidAnalyzer to rebuild all -HTML reports, useful after an new feature or a bug fix. +HTML reports, useful after an new feature or a bug fix. If you want to limit the +rebuild to a single day, a single month or year, you can use the --build_date +option by specifying the date part to rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy. The --preserve option should be used if you want to rotate your statistics and data. The value is the number of months to keep, older reports and data will be diff --git a/doc/squidanalyzer.3 b/doc/squidanalyzer.3 index 5d83c8e..df16a37 100644 --- a/doc/squidanalyzer.3 +++ b/doc/squidanalyzer.3 @@ -273,23 +273,27 @@ SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl script. Here are authorized usage: .PP .Vb 1 -\& squid\-analyzer [ \-c squidanalyzer.conf ] [\-l logfile] +\& Usage: squid\-analyzer [ \-c squidanalyzer.conf ] [\-l logfile] \& -\& \-c | \-\-configfile filename : path to the SquidAnalyzer configuration file. -\& By default: /etc/squidanalyzer.conf -\& \-d | \-\-debug : show debug informations. -\& \-h | \-\-help : show this message and exit. -\& \-l | \-\-logfile filename : path to the Squid logfile to parse. -\& By default: /var/log/squid/access.log -\& \-p | \-\-preserve number : used to set the statistic obsolescence in -\& number of month. Older stats will be removed. -\& \-r | \-\-rebuild : use this option to rebuild all html and graphs -\& output from all data files. -\& \-v | version : show version and exit. +\& \-c | \-\-configfile filename : path to the SquidAnalyzer configuration file. +\& By default: /etc/squidanalyzer.conf +\& \-b | \-\-build_date date : set the day to be rebuilt, format: yyyy\-mm\-dd, +\& yyyy\-mm or yyyy. Used with \-r or \-\-rebuild. +\& \-d | \-\-debug : show debug informations. +\& \-h | \-\-help : show this message and exit. +\& \-l | \-\-logfile filename : path to the Squid logfile to parse. +\& By default: /var/log/squid/access.log +\& \-p | \-\-preserve number : used to set the statistic obsolescence in +\& number of month. Older stats will be removed. +\& \-r | \-\-rebuild : use this option to rebuild all html and graphs +\& output from all data files. +\& \-v | version : show version and exit. .Ve .PP There is special options like \-\-rebuild that force SquidAnalyzer to rebuild all -\&\s-1HTML\s0 reports, useful after an new feature or a bug fix. +\&\s-1HTML\s0 reports, useful after an new feature or a bug fix. If you want to limit the +rebuild to a single day, a single month or year, you can use the \-\-build_date +option by specifying the date part to rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy. .PP The \-\-preserve option should be used if you want to rotate your statistics and data. The value is the number of months to keep, older reports and data will be diff --git a/squid-analyzer b/squid-analyzer index bd537b3..402e9e3 100644 --- a/squid-analyzer +++ b/squid-analyzer @@ -17,10 +17,12 @@ my $rebuild = ''; my $preserve = ''; my $debug = 0; my $version = 0; +my $build_date = ''; # get the command line parameters my $result = GetOptions ( "c|configfile=s" => \$configfile, + "b|build_date=s" => \$build_date, "d|debug!" => \$debug, "h|help" => \$help, "l|logfile=s" => \$logfile, @@ -35,6 +37,14 @@ if ($version) { exit 0; } +if ($build_date) { + $rebuild = 1; + if ( ($build_date !~ /^\d{4}-\d{2}-\d{2}$/) && ($build_date !~ /^\d{4}-\d{2}$/) && ($build_date !~ /^\d{4}$/) ) { + die("FATAL: bad syntax for build_date, expecting format: yyyy-mm-dd, yyyy-mm or yyyy\n"); + exit 0; + } +} + # Allow backward compatibility with release < 4.0 $configfile = $ARGV[0] if (($#ARGV == 0) && $ARGV[0]); if (($#ARGV < 0) && -e $DEFAULT_CONFFILE) { @@ -47,19 +57,21 @@ if (!$configfile || $help) { } # Instanciate SquidAnalyzer.pm perl module -my $sa = new SquidAnalyzer($configfile, $logfile, $debug); +my $sa = new SquidAnalyzer($configfile, $logfile, $debug, $rebuild); -# Run parsing -$sa->parseFile(); +# Run parsing only if we have a log file or that we are not in rebuild mode +$sa->parseFile() if (!$rebuild || $logfile); # Remove old statistics if ($preserve) { $sa->{preserve} = $preserve; } -# Recover month and year statistics from day stats +# In rebuild mode history time is not use and we must store the +# specific rebuild date if any is provided at command line. if ($rebuild) { $sa->{history_time} = ''; + $sa->{build_date} = $build_date; } # Generate graphics and html @@ -75,6 +87,8 @@ Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] -c | --configfile filename : path to the SquidAnalyzer configuration file. By default: /etc/squidanalyzer.conf + -b | --build_date date : set the date to be rebuilt, format: yyyy-mm-dd + or yyyy-mm or yyyy. Used with -r or --rebuild. -d | --debug : show debug informations. -h | --help : show this message and exit. -l | --logfile filename : path to the Squid logfile to parse.
$Translate{'User'}$Translate{'Count'}