tyler.durden/squidanalyzer
1
0
Fork 0
mirror of https://github.com/darold/squidanalyzer.git synced 2025-07-30 01:14:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add -b | --build_date command line option to limit rebuilt to the given date, the format of its value can be yyyy-mm-dd, yyyy-mm or yyyy. This should be used to prevent rebuilding all html file from all data file.

Browse Source
This commit is contained in:
Darold Gilles 2013-05-27 22:27:18 +02:00
parent 33700abc47
commit a3c6807ffd
5 changed files with 113 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README
View File

@ -131,22 +131,27 @@ USAGE
SquidAnalyzer can be run manually or by cron job using the SquidAnalyzer can be run manually or by cron job using the
squid-analyzer Perl script. Here are authorized usage: squid-analyzer Perl script. Here are authorized usage:
squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile]
-c | --configfile filename : path to the SquidAnalyzer configuration file. -c | --configfile filename : path to the SquidAnalyzer configuration file.
By default: /etc/squidanalyzer.conf By default: /etc/squidanalyzer.conf
-d | --debug : show debug informations. -b | --build_date date : set the day to be rebuilt, format: yyyy-mm-dd,
-h | --help : show this message and exit. yyyy-mm or yyyy. Used with -r or --rebuild.
-l | --logfile filename : path to the Squid logfile to parse. -d | --debug : show debug informations.
By default: /var/log/squid/access.log -h | --help : show this message and exit.
-p | --preserve number : used to set the statistic obsolescence in -l | --logfile filename : path to the Squid logfile to parse.
number of month. Older stats will be removed. By default: /var/log/squid/access.log
-r | --rebuild : use this option to rebuild all html and graphs -p | --preserve number : used to set the statistic obsolescence in
output from all data files. number of month. Older stats will be removed.
-v | version : show version and exit. -r | --rebuild : use this option to rebuild all html and graphs
output from all data files.
-v | version : show version and exit.
There is special options like --rebuild that force SquidAnalyzer to There is special options like --rebuild that force SquidAnalyzer to
rebuild all HTML reports, useful after an new feature or a bug fix. rebuild all HTML reports, useful after an new feature or a bug fix. If
you want to limit the rebuild to a single day, a single month or year,
you can use the --build_date option by specifying the date part to
rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy.
The --preserve option should be used if you want to rotate your The --preserve option should be used if you want to rotate your
statistics and data. The value is the number of months to keep, older statistics and data. The value is the number of months to keep, older

57
SquidAnalyzer.pm
View File

@ -16,7 +16,7 @@ use strict; # make things properly
BEGIN { BEGIN {
use Exporter(); use Exporter();
use vars qw($VERSION $COPYRIGHT $AUTHOR @ISA @EXPORT $ZCAT_PROG $BZCAT_PROG $RM_PROG); use vars qw($VERSION $COPYRIGHT $AUTHOR @ISA @EXPORT $ZCAT_PROG $BZCAT_PROG $RM_PROG);
use POSIX; use POSIX qw/ strftime /;
use IO::File; use IO::File;
# Set all internal variable # Set all internal variable
@ -127,19 +127,21 @@ my %Translate = (
'Click_year_stat' => 'Click on year\'s statistics link for details', 'Click_year_stat' => 'Click on year\'s statistics link for details',
'Mime_graph_hits_title' => 'Mime Type Hits Statistics on', 'Mime_graph_hits_title' => 'Mime Type Hits Statistics on',
'Mime_graph_bytes_title' => 'Mime Type Bytes Statistiques on', 'Mime_graph_bytes_title' => 'Mime Type Bytes Statistiques on',
'User' => 'User',
'Count' => 'Count',
); );
sub new sub new
{ {
my ($class, $conf_file, $log_file, $debug) = @_; my ($class, $conf_file, $log_file, $debug, $rebuild) = @_;
# Construct the class # Construct the class
my $self = {}; my $self = {};
bless $self, $class; bless $self, $class;
# Initialize all variables # Initialize all variables
$self->_init($conf_file, $log_file, $debug); $self->_init($conf_file, $log_file, $debug, $rebuild);
# Return the instance # Return the instance
return($self); return($self);
@ -184,6 +186,7 @@ sub parseFile
my $line_count = 0; my $line_count = 0;
my $line_processed_count = 0; my $line_processed_count = 0;
my $line_stored_count = 0; my $line_stored_count = 0;
# Read and parse each line of the access log file # Read and parse each line of the access log file
while ($line = <$logfile>) { while ($line = <$logfile>) {
chomp($line); chomp($line);
@ -382,7 +385,7 @@ sub _clear_stats
sub _init sub _init
{ {
my ($self, $conf_file, $log_file, $debug) = @_; my ($self, $conf_file, $log_file, $debug, $rebuild) = @_;
# Prevent for a call without instance # Prevent for a call without instance
if (!ref($self)) { if (!ref($self)) {
@ -398,7 +401,7 @@ sub _init
$conf_file = 'squidanalyzer.conf'; $conf_file = 'squidanalyzer.conf';
} }
} }
my %options = &parse_config($conf_file, $log_file); my %options = &parse_config($conf_file, $log_file, $rebuild);
# Use squid log file given as command line parameter # Use squid log file given as command line parameter
$options{LogFile} = $log_file if ($log_file); $options{LogFile} = $log_file if ($log_file);
@ -432,13 +435,13 @@ sub _init
die "ERROR: 'Output' configuration option must be set.\n"; die "ERROR: 'Output' configuration option must be set.\n";
} }
if (! -d $self->{Output}) { if (! -d $self->{Output}) {
die "ERROR: 'Output' dorectory $self->{Output} doesn't exists.\n"; die "ERROR: 'Output' directory $self->{Output} doesn't exists.\n";
} }
$self->{LogFile} = $options{LogFile} || '/var/log/squid/access.log'; $self->{LogFile} = $options{LogFile} || '/var/log/squid/access.log';
if (!$self->{LogFile}) { if (!$self->{LogFile}) {
die "ERROR: 'LogFile' configuration option must be set.\n"; die "ERROR: 'LogFile' configuration option must be set.\n";
} }
if (! -e $self->{LogFile}) { if (!$rebuild && ! -e $self->{LogFile}) {
die "ERROR: 'LogFile' $self->{LogFile} doesn't exists.\n"; die "ERROR: 'LogFile' $self->{LogFile} doesn't exists.\n";
} }
$self->{OrderUser} = lc($options{OrderUser}) || 'bytes'; $self->{OrderUser} = lc($options{OrderUser}) || 'bytes';
@ -507,7 +510,7 @@ sub _init
} }
# Get the last parsing date for incremental parsing # Get the last parsing date for incremental parsing
if (-e "$self->{Output}/SquidAnalyzer.current") { if (!$rebuild && -e "$self->{Output}/SquidAnalyzer.current") {
my $current = new IO::File; my $current = new IO::File;
unless($current->open("$self->{Output}/SquidAnalyzer.current")) { unless($current->open("$self->{Output}/SquidAnalyzer.current")) {
print STDERR "ERROR: Can't read file $self->{Output}/SquidAnalyzer.current, $!\n" if (!$self->{QuietMode}); print STDERR "ERROR: Can't read file $self->{Output}/SquidAnalyzer.current, $!\n" if (!$self->{QuietMode});
@ -711,7 +714,7 @@ sub _save_stat
my $path = join('/', $year, $month, $day); my $path = join('/', $year, $month, $day);
$path =~ s/[\/]+$//; $path =~ s/[\/]+$//;
#### Load history #### Load history if we are not rebuilding a particular day
if ($type eq 'day') { if ($type eq 'day') {
foreach my $d ("01" .. "31") { foreach my $d ("01" .. "31") {
$self->_read_stat($year, $month, $d, 'day'); $self->_read_stat($year, $month, $d, 'day');
@ -1167,6 +1170,25 @@ sub _print_footer
} }
sub check_build_date
{
my ($self, $year, $month, $day) = @_;
return 0 if (!$self->{build_date});
my ($y, $m, $d) = split(/\-/, $self->{build_date});
return 1 if ($year ne $y);
if ($m) {
return 1 if ($month && ($month ne $m));
if ($d) {
return 1 if ($day && ($day ne $d));
}
}
return 0;
}
sub buildHTML sub buildHTML
{ {
my ($self, $outdir) = @_; my ($self, $outdir) = @_;
@ -1204,6 +1226,7 @@ sub buildHTML
closedir DIR; closedir DIR;
foreach my $y (sort {$a <=> $b} @years) { foreach my $y (sort {$a <=> $b} @years) {
next if (!$y); next if (!$y);
next if ($self->check_build_date($y));
# Remove the full year repository if it is older that the last date to preserve # Remove the full year repository if it is older that the last date to preserve
if ($p_year && ($y < $p_year)) { if ($p_year && ($y < $p_year)) {
print STDERR "Removing obsolete statistics for year $y\n" if (!$self->{QuietMode}); print STDERR "Removing obsolete statistics for year $y\n" if (!$self->{QuietMode});
@ -1216,6 +1239,7 @@ sub buildHTML
closedir DIR; closedir DIR;
foreach my $m (sort {$a <=> $b} @months) { foreach my $m (sort {$a <=> $b} @months) {
next if (!$m); next if (!$m);
next if ($self->check_build_date($y, $m));
# Remove the full month repository if it is older that the last date to preserve # Remove the full month repository if it is older that the last date to preserve
if ($p_year && ("$y$m" < "$p_year$p_month")) { if ($p_year && ("$y$m" < "$p_year$p_month")) {
print STDERR "Removing obsolete statistics for month $y-$m\n" if (!$self->{QuietMode}); print STDERR "Removing obsolete statistics for month $y-$m\n" if (!$self->{QuietMode});
@ -1227,6 +1251,7 @@ sub buildHTML
my @days = grep { /^\d{2}$/ && -d "$outdir/$y/$m/$_"} readdir(DIR); my @days = grep { /^\d{2}$/ && -d "$outdir/$y/$m/$_"} readdir(DIR);
closedir DIR; closedir DIR;
foreach my $d (sort {$a <=> $b} @days) { foreach my $d (sort {$a <=> $b} @days) {
next if ($self->check_build_date($y, $m, $d));
next if ("$y$m$d" < "$old_year$old_month$old_day"); next if ("$y$m$d" < "$old_year$old_month$old_day");
print STDERR "Generating daily statistics for day $y-$m-$d\n" if (!$self->{QuietMode}); print STDERR "Generating daily statistics for day $y-$m-$d\n" if (!$self->{QuietMode});
$self->gen_html_output($outdir, $y, $m, $d); $self->gen_html_output($outdir, $y, $m, $d);
@ -2479,7 +2504,7 @@ sub _print_top_url_stat
if (exists $url_stat{$u}{users}) { if (exists $url_stat{$u}{users}) {
print $out qq{ print $out qq{
<div class="tooltipLink"><span class="information"><a href="http://$u/" target="_blank" class="domainLink">$u</a></span><div class="tooltip"> <div class="tooltipLink"><span class="information"><a href="http://$u/" target="_blank" class="domainLink">$u</a></span><div class="tooltip">
<table><tr><th>User</th><th>Count</th></tr> <table><tr><th>$Translate{'User'}</th><th>$Translate{'Count'}</th></tr>
}; };
my $k = 1; my $k = 1;
foreach my $user (sort { $url_stat{$u}{users}{$b} <=> $url_stat{$u}{users}{$a} } keys %{$url_stat{$u}{users}}) { foreach my $user (sort { $url_stat{$u}{users}{$b} <=> $url_stat{$u}{users}{$a} } keys %{$url_stat{$u}{users}}) {
@ -2711,7 +2736,7 @@ sub _print_top_domain_stat
if (exists $domain_stat{$u}{users}) { if (exists $domain_stat{$u}{users}) {
print $out qq{ print $out qq{
<div class="tooltipLink"><span class="information">*.$u</span><div class="tooltip"> <div class="tooltipLink"><span class="information">*.$u</span><div class="tooltip">
<table><tr><th>User</th><th>Count</th></tr> <table><tr><th>$Translate{'User'}</th><th>$Translate{'Count'}</th></tr>
}; };
my $k = 1; my $k = 1;
foreach my $user (sort { $domain_stat{$u}{users}{$b} <=> $domain_stat{$u}{users}{$a} } keys %{$domain_stat{$u}{users}}) { foreach my $user (sort { $domain_stat{$u}{users}{$b} <=> $domain_stat{$u}{users}{$a} } keys %{$domain_stat{$u}{users}}) {
@ -2867,7 +2892,7 @@ sub _gen_summary
sub parse_config sub parse_config
{ {
my ($file, $log_file) = @_; my ($file, $log_file, $rebuild) = @_;
die "FATAL: no configuration file!\n" if (!-e $file); die "FATAL: no configuration file!\n" if (!-e $file);
@ -2886,9 +2911,11 @@ sub parse_config
print STDERR "Error: you must give a valid output directory. See option: Output\n"; print STDERR "Error: you must give a valid output directory. See option: Output\n";
exit 0; exit 0;
} }
if (!$log_file && (!exists $opt{LogFile} || !-f $opt{LogFile})) { if (!$rebuild || $log_file) {
print STDERR "Error: you must give the path to the Squid log file. See option: LogFile\n"; if (!$log_file && (!exists $opt{LogFile} || !-f $opt{LogFile})) {
exit 0; print STDERR "Error: you must give the path to the Squid log file. See option: LogFile\n";
exit 0;
}
} }
if (exists $opt{DateFormat}) { if (exists $opt{DateFormat}) {
if ( ($opt{DateFormat} !~ m#\%y#) || (($opt{DateFormat} !~ m#\%m#) && ($opt{DateFormat} !~ m#\%M#) )|| ($opt{DateFormat} !~ m#\%d#) ) { if ( ($opt{DateFormat} !~ m#\%y#) || (($opt{DateFormat} !~ m#\%m#) && ($opt{DateFormat} !~ m#\%M#) )|| ($opt{DateFormat} !~ m#\%d#) ) {

31
doc/SquidAnalyzer.pod
View File

@ -136,22 +136,27 @@ match your network and file system configuration.
SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl
script. Here are authorized usage: script. Here are authorized usage:
squid-analyzer [ -c squidanalyzer.conf ] [-l logfile] Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile]
-c | --configfile filename : path to the SquidAnalyzer configuration file.
By default: /etc/squidanalyzer.conf
-b | --build_date date : set the day to be rebuilt, format: yyyy-mm-dd,
yyyy-mm or yyyy. Used with -r or --rebuild.
-d | --debug : show debug informations.
-h | --help : show this message and exit.
-l | --logfile filename : path to the Squid logfile to parse.
By default: /var/log/squid/access.log
-p | --preserve number : used to set the statistic obsolescence in
number of month. Older stats will be removed.
-r | --rebuild : use this option to rebuild all html and graphs
output from all data files.
-v | version : show version and exit.
-c | --configfile filename : path to the SquidAnalyzer configuration file.
By default: /etc/squidanalyzer.conf
-d | --debug : show debug informations.
-h | --help : show this message and exit.
-l | --logfile filename : path to the Squid logfile to parse.
By default: /var/log/squid/access.log
-p | --preserve number : used to set the statistic obsolescence in
number of month. Older stats will be removed.
-r | --rebuild : use this option to rebuild all html and graphs
output from all data files.
-v | version : show version and exit.
There is special options like --rebuild that force SquidAnalyzer to rebuild all There is special options like --rebuild that force SquidAnalyzer to rebuild all
HTML reports, useful after an new feature or a bug fix. HTML reports, useful after an new feature or a bug fix. If you want to limit the
rebuild to a single day, a single month or year, you can use the --build_date
option by specifying the date part to rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy.
The --preserve option should be used if you want to rotate your statistics and The --preserve option should be used if you want to rotate your statistics and
data. The value is the number of months to keep, older reports and data will be data. The value is the number of months to keep, older reports and data will be

30
doc/squidanalyzer.3
View File

@ -273,23 +273,27 @@ SquidAnalyzer can be run manually or by cron job using the squid-analyzer Perl
script. Here are authorized usage: script. Here are authorized usage:
.PP .PP
.Vb 1 .Vb 1
\& squid\-analyzer [ \-c squidanalyzer.conf ] [\-l logfile] \& Usage: squid\-analyzer [ \-c squidanalyzer.conf ] [\-l logfile]
\& \&
\& \-c | \-\-configfile filename : path to the SquidAnalyzer configuration file. \& \-c | \-\-configfile filename : path to the SquidAnalyzer configuration file.
\& By default: /etc/squidanalyzer.conf \& By default: /etc/squidanalyzer.conf
\& \-d | \-\-debug : show debug informations. \& \-b | \-\-build_date date : set the day to be rebuilt, format: yyyy\-mm\-dd,
\& \-h | \-\-help : show this message and exit. \& yyyy\-mm or yyyy. Used with \-r or \-\-rebuild.
\& \-l | \-\-logfile filename : path to the Squid logfile to parse. \& \-d | \-\-debug : show debug informations.
\& By default: /var/log/squid/access.log \& \-h | \-\-help : show this message and exit.
\& \-p | \-\-preserve number : used to set the statistic obsolescence in \& \-l | \-\-logfile filename : path to the Squid logfile to parse.
\& number of month. Older stats will be removed. \& By default: /var/log/squid/access.log
\& \-r | \-\-rebuild : use this option to rebuild all html and graphs \& \-p | \-\-preserve number : used to set the statistic obsolescence in
\& output from all data files. \& number of month. Older stats will be removed.
\& \-v | version : show version and exit. \& \-r | \-\-rebuild : use this option to rebuild all html and graphs
\& output from all data files.
\& \-v | version : show version and exit.
.Ve .Ve
.PP .PP
There is special options like \-\-rebuild that force SquidAnalyzer to rebuild all There is special options like \-\-rebuild that force SquidAnalyzer to rebuild all
\&\s-1HTML\s0 reports, useful after an new feature or a bug fix. \&\s-1HTML\s0 reports, useful after an new feature or a bug fix. If you want to limit the
rebuild to a single day, a single month or year, you can use the \-\-build_date
option by specifying the date part to rebuild, format: yyyy-mm-dd, yyyy-mm or yyyy.
.PP .PP
The \-\-preserve option should be used if you want to rotate your statistics and The \-\-preserve option should be used if you want to rotate your statistics and
data. The value is the number of months to keep, older reports and data will be data. The value is the number of months to keep, older reports and data will be

22
squid-analyzer
View File

@ -17,10 +17,12 @@ my $rebuild = '';
my $preserve = ''; my $preserve = '';
my $debug = 0; my $debug = 0;
my $version = 0; my $version = 0;
my $build_date = '';
# get the command line parameters # get the command line parameters
my $result = GetOptions ( my $result = GetOptions (
"c|configfile=s" => \$configfile, "c|configfile=s" => \$configfile,
"b|build_date=s" => \$build_date,
"d|debug!" => \$debug, "d|debug!" => \$debug,
"h|help" => \$help, "h|help" => \$help,
"l|logfile=s" => \$logfile, "l|logfile=s" => \$logfile,
@ -35,6 +37,14 @@ if ($version) {
exit 0; exit 0;
} }
if ($build_date) {
$rebuild = 1;
if ( ($build_date !~ /^\d{4}-\d{2}-\d{2}$/) && ($build_date !~ /^\d{4}-\d{2}$/) && ($build_date !~ /^\d{4}$/) ) {
die("FATAL: bad syntax for build_date, expecting format: yyyy-mm-dd, yyyy-mm or yyyy\n");
exit 0;
}
}
# Allow backward compatibility with release < 4.0 # Allow backward compatibility with release < 4.0
$configfile = $ARGV[0] if (($#ARGV == 0) && $ARGV[0]); $configfile = $ARGV[0] if (($#ARGV == 0) && $ARGV[0]);
if (($#ARGV < 0) && -e $DEFAULT_CONFFILE) { if (($#ARGV < 0) && -e $DEFAULT_CONFFILE) {
@ -47,19 +57,21 @@ if (!$configfile || $help) {
} }
# Instanciate SquidAnalyzer.pm perl module # Instanciate SquidAnalyzer.pm perl module
my $sa = new SquidAnalyzer($configfile, $logfile, $debug); my $sa = new SquidAnalyzer($configfile, $logfile, $debug, $rebuild);
# Run parsing # Run parsing only if we have a log file or that we are not in rebuild mode
$sa->parseFile(); $sa->parseFile() if (!$rebuild || $logfile);
# Remove old statistics # Remove old statistics
if ($preserve) { if ($preserve) {
$sa->{preserve} = $preserve; $sa->{preserve} = $preserve;
} }
# Recover month and year statistics from day stats # In rebuild mode history time is not use and we must store the
# specific rebuild date if any is provided at command line.
if ($rebuild) { if ($rebuild) {
$sa->{history_time} = ''; $sa->{history_time} = '';
$sa->{build_date} = $build_date;
} }
# Generate graphics and html # Generate graphics and html
@ -75,6 +87,8 @@ Usage: squid-analyzer [ -c squidanalyzer.conf ] [-l logfile]
-c | --configfile filename : path to the SquidAnalyzer configuration file. -c | --configfile filename : path to the SquidAnalyzer configuration file.
By default: /etc/squidanalyzer.conf By default: /etc/squidanalyzer.conf
-b | --build_date date : set the date to be rebuilt, format: yyyy-mm-dd
or yyyy-mm or yyyy. Used with -r or --rebuild.
-d | --debug : show debug informations. -d | --debug : show debug informations.
-h | --help : show this message and exit. -h | --help : show this message and exit.
-l | --logfile filename : path to the Squid logfile to parse. -l | --logfile filename : path to the Squid logfile to parse.