From 86f8f6a53062cc015122bba3e16679538313807f Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Fri, 12 Sep 2025 04:30:12 +0000 Subject: [PATCH] Bug 30614: [alternate] Fallback to a GET request if HEAD returns error status This enhancement makes a GET request as a fallback if HEAD returns an error status using the existing AnyEvent module to maintain async checking of URLs. This doesn't fix all of the errors produced by the sample error MARC attached - any further support is welcomed. This patch adds a --where option to limit the biblios checked, and an extra level of verbosity to see the full response To test: 1. Download MARC. Go to Cataloguing in the staff interface and stage the MARC file for import, then import. 2. Run the URL check job and confirm erroring URLs perl misc/cronjobs/check-url-quick.pl -v > test.txt grep -E "tand|wiley|natlib|cold|canterbury" test.txt 3. Apply patch and restart services 4. Repeat step 2 and notice 2 of the URLs are now returning 200 OK status codes. Sponsored-by: Earth Sciences New Zealand --- misc/cronjobs/check-url-quick.pl | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/misc/cronjobs/check-url-quick.pl b/misc/cronjobs/check-url-quick.pl index 78b8adc229a..2210cef6991 100755 --- a/misc/cronjobs/check-url-quick.pl +++ b/misc/cronjobs/check-url-quick.pl @@ -32,10 +32,12 @@ my ( $verbose, $help, $html ) = ( 0, 0, 0 ); my ( $host, $host_intranet ) = ( '', '' ); my ( $timeout, $maxconn ) = ( 10, 200 ); my @tags; -my $uriedit = "/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber="; -my $user_agent = 'Mozilla/5.0 (compatible; U; Koha checkurl)'; +my $uriedit = "/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber="; +my $user_agent = 'Mozilla/5.0 (compatible; U; Koha checkurl)'; +my $whereclause = ''; GetOptions( - 'verbose' => \$verbose, + 'verbose+' => \$verbose, + 'where:s' => \$whereclause, 'html' => \$html, 'h|help' => \$help, 'host=s' => \$host, @@ -81,7 +83,10 @@ sub report { # Check all URLs from all current Koha biblio records sub check_all_url { - my $sth = C4::Context->dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber"); + if ($whereclause) { + $whereclause = "WHERE $whereclause"; + } + my $sth = C4::Context->dbh->prepare("SELECT biblionumber FROM biblioitems $whereclause ORDER BY biblionumber"); $sth->execute; my $count = 0; # Number of requested URL @@ -108,8 +113,19 @@ sub check_all_url { sub { my ( undef, $hdr ) = @_; $count--; - report( $hdr, $biblionumber, $url ) - if $hdr->{Status} !~ /^2/ || $verbose; + if ( $hdr->{Status} !~ /^2/ ) { + http_request( + GET => $url, + headers => { 'user-agent' => $user_agent }, + timeout => $timeout, + sub { + my ( undef, $hdr ) = @_; + report( $hdr, $biblionumber, $url ) + if $hdr->{Status} !~ /^2/ || $verbose; + print Data::Dumper::Dumper($hdr) if $verbose > 1; + } + ); + } }, ); } -- 2.39.5