490 lines
12 KiB
Perl
Executable File
490 lines
12 KiB
Perl
Executable File
##################################################################################
|
|
# Goliath Tools LWP Module
|
|
##################################################################################
|
|
# Copyright (c) 2007-2023 Pandora FMS
|
|
# This code is not free or OpenSource. Please don't redistribute.
|
|
##################################################################################
|
|
|
|
package PandoraFMS::Goliat::GoliatLWP;
|
|
|
|
use PandoraFMS::Goliat::GoliatTools;
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Data::Dumper;
|
|
|
|
use IO::Socket::INET6;
|
|
use LWP::UserAgent;
|
|
use LWP::ConnCache;
|
|
use HTTP::Request::Common;
|
|
use HTTP::Response;
|
|
use HTML::TreeBuilder;
|
|
use HTML::Element;
|
|
use HTTP::Cookies;
|
|
use URI::URL;
|
|
use Time::Local;
|
|
use Time::HiRes qw ( gettimeofday );
|
|
|
|
# For IPv6 support in Net::HTTP.
|
|
BEGIN {
|
|
$Net::HTTP::SOCKET_CLASS = 'IO::Socket::INET6';
|
|
require Net::HTTP;
|
|
}
|
|
|
|
# Japanese encoding support
|
|
use Encode::Guess qw/euc-jp shiftjis iso-2022-jp/;
|
|
|
|
require Exporter;
|
|
|
|
our @ISA = ("Exporter");
|
|
our %EXPORT_TAGS = ( 'all' => [ qw() ] );
|
|
our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
|
|
our @EXPORT = qw(
|
|
g_http_task
|
|
@task_requests
|
|
@task_reqsec
|
|
@task_fails
|
|
@task_time
|
|
@task_end
|
|
@task_sessions
|
|
@task_ssec
|
|
@task_get_string
|
|
@task_get_content
|
|
@task_session_fails
|
|
);
|
|
|
|
our @task_requests;
|
|
our @task_reqsec;
|
|
our @task_fails;
|
|
our @task_time;
|
|
our @task_end;
|
|
our @task_sessions;
|
|
our @task_ssec;
|
|
our @task_get_string;
|
|
our @task_get_content;
|
|
our @task_session_fails;
|
|
our $goliat_abort;
|
|
|
|
sub parse_html ($;$)
|
|
{
|
|
my $p = $_[1];
|
|
$p = _new_tree_maker() unless $p;
|
|
$p->parse($_[0]);
|
|
}
|
|
|
|
|
|
sub parse_htmlfile ($;$)
|
|
{
|
|
my($file, $p) = @_;
|
|
local(*HTML);
|
|
open(HTML, $file) or return undef;
|
|
$p = _new_tree_maker() unless $p;
|
|
$p->parse_file(\*HTML);
|
|
}
|
|
|
|
sub _new_tree_maker
|
|
{
|
|
my $p = HTML::TreeBuilder->new(implicit_tags => 1,
|
|
ignore_unknown => 1,
|
|
ignore_text => 0,
|
|
'warn' => 0,
|
|
);
|
|
$p->strict_comment(1);
|
|
$p;
|
|
}
|
|
|
|
|
|
sub g_http_task {
|
|
my ( $config, $thread_id, @work_list ) = @_;
|
|
my ( $ax, $bx, $cx ); # used in FOR loop
|
|
my ( $ttime1, $ttime2, $ttime_tot );
|
|
|
|
my $resp; # HTTP Response
|
|
my $total_requests = 0;
|
|
my $total_valid_requests = 0;
|
|
my $total_invalid_request = 0;
|
|
my $cookie_file = "/tmp/gtc_".$thread_id."_".g_trash_ascii (3);
|
|
my $check_string = 1;
|
|
my $get_string = "";
|
|
my $get_content = "";
|
|
my $get_content_advanced = "";
|
|
|
|
my $ua = new LWP::UserAgent;
|
|
if (!defined($ua)) {
|
|
die("LWP::UserAgent->new() failed. Not enough memory?");
|
|
}
|
|
$task_requests [$thread_id] = 0 ;
|
|
$task_sessions [$thread_id] = 0 ;
|
|
$task_reqsec[$thread_id] = 0;
|
|
$task_fails[$thread_id] = 0;
|
|
$task_session_fails[$thread_id] = 0;
|
|
$task_ssec[$thread_id] = 0;
|
|
$task_end[$thread_id] = 0;
|
|
$task_time[$thread_id] = 0;
|
|
$task_get_string[$thread_id] = "";
|
|
$task_get_content[$thread_id] = "";
|
|
|
|
$ua->agent($config->{"agent"});
|
|
$ua->protocols_allowed( ['http', 'https'] );
|
|
$ua->default_headers->push_header('pragma' => "no-cache");
|
|
$ua->timeout ($config->{"timeout"});
|
|
$ua->max_size($config->{"maxsize"});
|
|
$ua->use_alarm($config->{"alarm"});
|
|
|
|
# Disable SSL certificate host verification
|
|
if ($ua->can ('ssl_opts')) {
|
|
$ua->ssl_opts("verify_hostname" => 0);
|
|
}
|
|
|
|
# Set proxy
|
|
|
|
if ($config->{'proxy'} ne ""){
|
|
$ua->proxy(['http','https'], $config->{'proxy'});
|
|
}
|
|
|
|
# Set HTTP Proxy auth
|
|
if ($config->{'auth_user'} ne "") {
|
|
$ua->credentials(
|
|
$config->{'auth_server'},
|
|
$config->{'auth_realm'},
|
|
$config->{'auth_user'} => $config->{'auth_pass'} );
|
|
}
|
|
|
|
if ( -e $cookie_file){
|
|
unlink ($cookie_file);
|
|
}
|
|
my $cookies = HTTP::Cookies->new ('file' => $cookie_file, 'autosave' => '0');
|
|
|
|
$ttime1 = Time::HiRes::gettimeofday();
|
|
for ($ax = 0; $ax != $config->{'retries'}; $ax++){
|
|
for ($bx = 0; $bx < $config->{"work_items"}; $bx++){
|
|
if ($config->{'con_delay'} > 0){
|
|
sleep ($config->{'con_delay'});
|
|
}
|
|
$total_requests++;
|
|
# Start to count!
|
|
$check_string = 1;
|
|
# Prepare parameters
|
|
my $params = "";
|
|
$cx = 0;
|
|
while (defined($work_list[$bx]->{'variable_name'}[$cx])){
|
|
if ($cx > 0){
|
|
$params = $params."&";
|
|
}
|
|
$params = $params . $work_list[$bx]->{'variable_name'}[$cx] . "=" . $work_list[$bx]->{'variable_value'}[$cx];
|
|
$cx++;
|
|
}
|
|
|
|
if ( (defined($work_list[$bx]->{'http_auth_realm'})) && (defined($work_list[$bx]->{'http_auth_serverport'}))&& (defined($work_list[$bx]->{'http_auth_user'})) && (defined($work_list[$bx]->{'http_auth_pass'}))) {
|
|
if ($work_list[$bx]->{'http_auth_realm'} ne "") {
|
|
$ua->credentials(
|
|
$work_list[$bx]->{'http_auth_serverport'},
|
|
$work_list[$bx]->{'http_auth_realm'},
|
|
$work_list[$bx]->{'http_auth_user'} => $work_list[$bx]->{'http_auth_pass'}
|
|
);
|
|
}
|
|
}
|
|
|
|
# GET
|
|
if ($work_list[$bx]->{'type'} eq "GET"){
|
|
if ($cx > 0){
|
|
$params = $work_list[$bx]->{'url'} . "?" . $params;
|
|
} else {
|
|
$params = $work_list[$bx]->{'url'};
|
|
}
|
|
$resp = g_get_page ( $ua, $params, $work_list[$bx]->{'headers'}, $work_list[$bx]->{'debug'});
|
|
|
|
# POST
|
|
} elsif ($work_list[$bx]->{'type'} eq "POST") {
|
|
$resp = g_post_page ( $ua, $work_list[$bx]->{'url'}, $params, $work_list[$bx]->{'headers'}, $work_list[$bx]->{'debug'});
|
|
|
|
# HEAD
|
|
} else {
|
|
if ($cx > 0){
|
|
$params = $work_list[$bx]->{'url'} . "?" . $params;
|
|
} else {
|
|
$params = $work_list[$bx]->{'url'};
|
|
}
|
|
$resp = g_head_page ( $ua, $params, $work_list[$bx]->{'headers'}, $work_list[$bx]->{'debug'});
|
|
}
|
|
|
|
# Check for errors.
|
|
if ($resp->code() == 500) {
|
|
$total_invalid_request++;
|
|
$bx = $config->{"work_items"};
|
|
$check_string=0;
|
|
last;
|
|
}
|
|
|
|
# Get string ?
|
|
if (defined($work_list[$bx]->{'get_string'})) {
|
|
my $as_string = $resp->as_string;
|
|
my $temp = $work_list[$bx]->{'get_string'};
|
|
if ($as_string =~ m/($temp)/) {
|
|
$task_get_string[$thread_id] = $1;
|
|
}
|
|
}
|
|
|
|
# Get response ?
|
|
if ($work_list[$bx]->{'get_content_advanced'} ne "") {
|
|
my $content = $resp->decoded_content;
|
|
my $temp = $work_list[$bx]->{'get_content_advanced'};
|
|
if ($content =~ m/$temp/) {
|
|
$task_get_content[$thread_id] = $1 if defined ($1);
|
|
}
|
|
} elsif ($work_list[$bx]->{'get_content'} ne "") {
|
|
my $content = $resp->decoded_content;
|
|
my $temp = $work_list[$bx]->{'get_content'};
|
|
if ($content =~ m/($temp)/) {
|
|
$task_get_content[$thread_id] = $1;
|
|
}
|
|
} else {
|
|
$task_get_content[$thread_id] = $resp->decoded_content;
|
|
}
|
|
|
|
# Resource bashing
|
|
if ((defined($work_list[$bx]->{'get_resources'})) && ($work_list[$bx]->{'get_resources'} == 1)){
|
|
$total_requests = g_get_all_links ($config, $ua, $resp, $total_requests, $work_list[$bx]->{'url'}, $work_list[$bx]->{'headers'}, $work_list[$bx]->{'debug'});
|
|
}
|
|
|
|
# CHECKSTRING check
|
|
$cx = 0;
|
|
while (defined($work_list[$bx]->{'checkstring'}[$cx])) {
|
|
my $match_string = $work_list[$bx]->{'checkstring'}[$cx];
|
|
my $as_string = $resp->as_string;
|
|
|
|
my $guess = Encode::Guess::guess_encoding($as_string);
|
|
if (ref $guess) {
|
|
$as_string = $guess->decode($as_string);
|
|
}
|
|
unless (utf8::is_utf8($match_string)) {
|
|
utf8::decode($match_string);
|
|
}
|
|
|
|
if ( $as_string =~ m/$match_string/i ){
|
|
$total_valid_requests++;
|
|
} else {
|
|
$total_invalid_request++;
|
|
$bx = $config->{"work_items"}; # Abort session remaining request
|
|
$check_string=0;
|
|
}
|
|
$cx++;
|
|
}
|
|
|
|
# CHECKNOTSTRING check
|
|
$cx = 0;
|
|
while (defined($work_list[$bx]->{'checknotstring'}[$cx])) {
|
|
my $match_string = $work_list[$bx]->{'checknotstring'}[$cx];
|
|
my $as_string = $resp->as_string;
|
|
|
|
my $guess = Encode::Guess::guess_encoding($as_string);
|
|
if (ref $guess) {
|
|
$as_string = $guess->decode($as_string);
|
|
}
|
|
unless (utf8::is_utf8($match_string)) {
|
|
utf8::decode($match_string);
|
|
}
|
|
|
|
if ( $as_string !~ m/$match_string/i ){
|
|
$total_valid_requests++;
|
|
} else {
|
|
$total_invalid_request++;
|
|
$bx = $config->{"work_items"}; # Abort session remaining request
|
|
$check_string=0;
|
|
}
|
|
$cx++;
|
|
}
|
|
|
|
# Cookie carry on
|
|
if (defined ($work_list[$bx]->{'cookie'}) && $work_list[$bx]->{'cookie'} == 1){
|
|
$cookies->extract_cookies($resp);
|
|
$ua->cookie_jar($cookies);
|
|
}
|
|
|
|
# End just now by pressing CTRL-C or Kill Signal !
|
|
#if ($goliat_abort == 1){
|
|
#$ax = $config->{'retries'};
|
|
#$bx = $config->{'items'};
|
|
#goto END_LOOP;
|
|
#}
|
|
} #main work_detail loop
|
|
$ttime2 = Time::HiRes::gettimeofday();
|
|
|
|
$ttime_tot = $ttime2 - $ttime1; # Total time for this task
|
|
$task_time[$thread_id] = $ttime_tot;
|
|
$task_requests [$thread_id] = $total_requests;
|
|
if ($ttime_tot > 0 ){
|
|
$task_reqsec[$thread_id] = $total_requests / $ttime_tot;
|
|
} else {
|
|
$task_reqsec[$thread_id] = $total_requests;
|
|
}
|
|
$task_fails[$thread_id] = $total_invalid_request;
|
|
if ($check_string == 0){
|
|
$task_session_fails[$thread_id]++
|
|
}
|
|
$task_sessions [$thread_id]++;
|
|
if ($task_sessions [$thread_id] > 0 ){
|
|
$task_ssec[$thread_id] = $ttime_tot / $task_sessions [$thread_id];
|
|
} else {
|
|
$task_ssec[$thread_id] = $task_sessions[$thread_id];
|
|
}
|
|
sleep $config->{'ses_delay'};
|
|
}
|
|
END_LOOP:
|
|
|
|
$cookies->clear;
|
|
|
|
if ( -f $cookie_file){
|
|
unlink ($cookie_file);
|
|
}
|
|
|
|
$task_end[$thread_id] = 1;
|
|
}
|
|
|
|
|
|
sub g_get_all_links {
|
|
my ($config, $ua, $response, $counter, $myurl, $headers, $debug) = @_;
|
|
my $html;
|
|
|
|
if ($response->is_success) {
|
|
$html = $response->content;
|
|
} else {
|
|
return $counter;
|
|
}
|
|
# Beware this funcion, needs to be destroyed after use it !!!
|
|
my $parsed_html = parse_html($html);
|
|
#$ua->conn_cache(LWP::ConnCache->new());
|
|
|
|
my @url_list;
|
|
my $url = "";
|
|
my $link;
|
|
my $full_url;
|
|
|
|
for (@{ $parsed_html->extract_links( ) }) {
|
|
$link=$_->[0];
|
|
if (($link =~ m/.png/i) || ($link =~ m/.gif/i) || ($link =~ m/.htm/i) ||
|
|
($link =~ m/.html/i) || ($link =~ m/.pdf/i) || ($link =~ m/.jpg/i)
|
|
|| ($link =~ m/.ico/i)){
|
|
$url = new URI::URL $link;
|
|
$full_url = $url->abs($myurl);
|
|
@url_list = $full_url;
|
|
}
|
|
|
|
}
|
|
$parsed_html->delete;
|
|
my $ax = 0;
|
|
while ($full_url = pop(@url_list)) {
|
|
g_get_page ($ua, $full_url, $headers, $debug);
|
|
$counter++;
|
|
$ax++;
|
|
if ($ax > $config->{"max_depth"}){
|
|
return $counter;
|
|
}
|
|
}
|
|
return $counter;
|
|
}
|
|
|
|
sub g_get_page {
|
|
my $ua = $_[0];
|
|
my $url = $_[1];
|
|
my $headers = $_[2];
|
|
my $debug = $_[3];
|
|
|
|
my $req = HTTP::Request->new(GET => $url);
|
|
if (!defined($req)) {
|
|
die("HTTP::Request->new() failed. Not enough memory?");
|
|
}
|
|
$req->header('Accept' => 'text/html');
|
|
while (my ($header, $value) = each %{$headers}) {
|
|
$req->header($header => $value);
|
|
}
|
|
my $response = $ua->request($req);
|
|
return $response if ($debug eq '');
|
|
|
|
# Debug
|
|
if (open (DEBUG, '>>', $debug . '.req')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $req->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
if (open (DEBUG, '>>', $debug . '.res')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $response->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
return $response;
|
|
}
|
|
|
|
sub g_head_page {
|
|
my $ua = $_[0];
|
|
my $url = $_[1];
|
|
my $headers = $_[2];
|
|
my $debug = $_[3];
|
|
|
|
my $req = HTTP::Request->new(HEAD => $url);
|
|
if (!defined($req)) {
|
|
die("HTTP::Request->new() failed. Not enough memory?");
|
|
}
|
|
$req->header('Accept' => 'text/html');
|
|
while (my ($header, $value) = each %{$headers}) {
|
|
$req->header($header => $value);
|
|
}
|
|
my $response = $ua->request($req);
|
|
return $response if ($debug eq '');
|
|
|
|
# Debug
|
|
if (open (DEBUG, '>>', $debug . '.req')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $req->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
if (open (DEBUG, '>>', $debug . '.res')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $response->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
return $response;
|
|
}
|
|
|
|
sub g_post_page {
|
|
my $ua = $_[0];
|
|
my $url = $_[1];
|
|
my $content = $_[2];
|
|
my $headers = $_[3];
|
|
my $debug = $_[4];
|
|
|
|
my $req = HTTP::Request->new(POST => $url);
|
|
$req->content_type('application/x-www-form-urlencoded');
|
|
$req->content ($content);
|
|
while (my ($header, $value) = each %{$headers}) {
|
|
$req->header($header => $value);
|
|
}
|
|
my $response = $ua->request($req);
|
|
return $response if ($debug eq '');
|
|
|
|
# Debug
|
|
if (open (DEBUG, '>>', $debug . '.req')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $req->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
if (open (DEBUG, '>>', $debug . '.res')) {
|
|
print DEBUG "[Goliat debug " . time () . "]\n";
|
|
print DEBUG $response->as_string ();
|
|
print "\n";
|
|
close (DEBUG);
|
|
}
|
|
return $response;
|
|
}
|
|
|
|
# End of function declaration
|
|
# End of defined Code
|
|
|
|
1;
|
|
__END__
|