From bbea9c030c5832f09862168f652e7bd07c793606 Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Thu, 12 Apr 2018 14:32:14 +0200 Subject: [PATCH] Bug 20570: Add script to complete scanned article requests Content-Type: text/plain; charset=utf-8 --- Koha/ArticleRequests.pm | 125 ++++++++++++++++++++++ misc/migration_tools/complete_article_requests.pl | 121 +++++++++++++++++++++ opac/svc/scan_requests | 92 ++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100755 misc/migration_tools/complete_article_requests.pl create mode 100755 opac/svc/scan_requests diff --git a/Koha/ArticleRequests.pm b/Koha/ArticleRequests.pm index bc47789..4edad89 100644 --- a/Koha/ArticleRequests.pm +++ b/Koha/ArticleRequests.pm @@ -20,8 +20,10 @@ package Koha::ArticleRequests; use Modern::Perl; use Carp; +use Data::Dumper qw/Dumper/; use Koha::Database; +use Koha::DateUtils qw/dt_from_string/; use Koha::ArticleRequest; use Koha::ArticleRequest::Status; @@ -82,6 +84,129 @@ sub canceled { return Koha::ArticleRequests->search( $params ); } +=head3 match_scans + + $class->match_scans({ uploadcategory => 'SCANS' }); + + Match all requests in processing stage with uploaded files in the + specified category submitted later than the first request was set + to processing. Save upload URL in the requests and mark complete. + + Groups of uploaded files submitted within the same interval as described + by the optional interval parameter (default 10) are considered as + batches to be sorted on filename. Disable this behavior by passing -1. + In that case the strict upload order is followed. + +=cut + +sub match_scans { + my ( $class, $params ) = @_; + my $category = $params->{uploadcategory}; + my $interval = $params->{interval} // 10; # minutes + + my $requests = $class->search_scans_in_process->as_list; + return if !@$requests; # no requests to match + + # Find time to start looking for uploads + my $first_processed = $requests->[0]->updated_on; + + #FIXME Replace with Koha::UploadedFiles + # Public should be true, since we will create OPAC URLs + my $uploads = C4::Context->dbh->selectall_arrayref("SELECT filename,hashvalue,dtcreated FROM uploaded_files WHERE uploadcategorycode=? AND dtcreated>? AND public=1 ORDER BY id", undef, ( $category, $first_processed )); + return if !@$uploads; + + # Split uploads in batches (based on interval). We can assume that the + # uploads have ascending timestamps. + my @batches; + my $n = 0; + my $dt_last; + foreach my $u (@$uploads) { + my $dt = dt_from_string( $u->[2], 'sql' ); + if( !defined $dt_last ) { + $dt_last = $dt; + } elsif( $dt_last->delta_ms( $dt )->in_units('minutes') > $interval ) { + # delta_ms returns non-negative values only + $dt_last = $dt; + push @batches, $n; + $n = 0; + } + $n++; + } + push @batches, $n if $n; + + # Rearrange order of uploads within a batch on filename + # This is done since the order of multiple selected files in the file + # upload control is not reliable (often in reverse). + my $pos = 0; + foreach my $s (@batches) { + @$uploads[$pos..$pos+$s-1] = sort { lc($a->[0]) cmp lc($b->[0]) } @$uploads[$pos..$pos+$s-1] if $s > 1; + $pos += $s; + } + + # Match while we have uploads or requests + my $result = {}; + while( @$uploads && @$requests ) { + my $upload = shift @$uploads; + my $req = shift @$requests; + my $url = C4::Context->preference('OPACBaseURL').'/cgi-bin/koha/opac-retrieve-file.pl?id='.$upload->[1]; + $req->urls($url)->store; + $req->complete; + } +} + +=head3 search_scans_in_process + + $class->search_scans_in_process({ list => 1 }); + + Search for article requests with status in processing, format SCAN + and empty urls field. Order by last change. + + The optional list parameter (used in opac/svc/scan_requests) makes + that an arrayref of request id's is returned. + Without it, a Koha::Objects search is returned (used by match_scans). + +=cut + +sub search_scans_in_process { + my ( $class, $params ) = @_; + my $req_in_process = $class->search( + { status => 'PROCESSING', format => 'SCAN', urls => [ '', undef ] }, + { order_by => 'updated_on' }, + ); + if( $params->{list} ) { + return [ map { $_->id } $req_in_process->as_list ]; + } + return $req_in_process; +} + +=head3 complete_scans + + Save URLs in scan requests and mark them as completed. + + The $scans parameter is a hashref of the form { request_id => url, ... } + + Returns the number of completed scan requests. + + Note: The routine is called by opac/svc/scan_requests?op=complete when + you pass the results of external matching (as opposed to match_scans) + back to Koha. This allows for customization with external web services. + +=cut + +sub complete_scans { + my ( $class, $scans ) = @_; + my $n = 0; + foreach my $id ( keys %$scans ) { + my $req = Koha::ArticleRequests->find( $id ); + if( $req ) { + $req->urls( $scans->{$id} )->store; + $req->complete; + $n++; + } + } + return $n; +} + =head3 _type =cut diff --git a/misc/migration_tools/complete_article_requests.pl b/misc/migration_tools/complete_article_requests.pl new file mode 100755 index 0000000..af5f749 --- /dev/null +++ b/misc/migration_tools/complete_article_requests.pl @@ -0,0 +1,121 @@ +#!/usr/bin/perl + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use C4::Context; +use Koha::ArticleRequests; + +my $params = {}; +GetOptions( + 'uploadcategory:s' => \$params->{uploadcategory}, + 'upload_interval:i' => \$params->{upload_interval}, + 'commit' => \$params->{commit}, + 'help' => \$params->{help}, + 'manual' => \$params->{manual}, +); + +if( $params->{help} ) { + pod2usage(); +} elsif( $params->{manual} ) { + pod2usage({ -verbose => 2 }); +} elsif( !C4::Context->preference('ArticleRequests') || + C4::Context->preference('ArticleRequestsSupportedFormats') !~ /SCAN/ ) { + print "First enable ArticleRequests and add SCAN as supported format.\n"; +} elsif( !$params->{uploadcategory} ) { + print "Please specify an uploadcategory.\n"; + pod2usage(); +} elsif( !$params->{commit} ) { + print "You need to specify the commit parameter.\n"; + pod2usage(); +} else { + Koha::ArticleRequests->match_scans({ uploadcategory => $params->{uploadcategory}, interval => $params->{upload_interval} }); +} + +=head1 NAME + +complete_article_requests.pl + +=head1 COPYRIGHT + +Copyright 2018 Rijksmuseum + +=head1 LICENSE + +This file is part of Koha. + +Koha is free software; you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +Koha is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with Koha; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 SYNOPSIS + + complete_article_requests.pl -help + complete_article_requests.pl -manual + complete_article_requests.pl -uploadcategory SCANS -commit + +=head1 DESCRIPTION + +This script serves to complete scanned article requests by matching them with +uploads in a specified upload category. + +How does it work? The script is based on the following criteria: + + [1] The order of article requests in processing stage is followed + according to the updated_on timestamp. + + [2] Only uploaded files in the specified category, created later than the + timestamp of the first processing request are considered. + + [3] The uploads should have been marked as Public. + + [4] The algorithm tries to correct multiple files submitted in reverse order + by sorting on filename per group of files uploaded within a specfied + number of minutes (upload_interval). If you set this number to -1, + the exact upload order will be used. + +In the process of linking files to requests, the urls field is filled and the +request is marked completed (triggering a mail to the requestor). + +=head1 OPTIONS + +=head2 help + +Print usage statement + +=head2 manual + +Print complete POD documentation + +=head2 uploadcategory + +Code of the Koha upload category that you are using for scanned article +request materials. + +=head2 upload_interval + +Optional parameter to specify within how many minutes a series of consecutive +uploads are considered as a batch or group to be sorted on filename. The +match_scans method has a fallback default value. + +=head1 ADDITIONAL COMMENTS + +Instead of using the Upload feature in connection with this script, Koha also +allows you to automate matching files with requests externally by offering +the script opac/svc/scan_requests with op=list and op=complete. That script +contains more documentation about its use. + +=head1 AUTHOR + +Marcel de Rooy, Rijksmuseum Amsterdam, The Netherlands + +=cut diff --git a/opac/svc/scan_requests b/opac/svc/scan_requests new file mode 100755 index 0000000..e1d6635 --- /dev/null +++ b/opac/svc/scan_requests @@ -0,0 +1,92 @@ +#!/usr/bin/perl + +use Modern::Perl; +use CGI qw ( -utf8 ); +use C4::Context; +use Koha::ArticleRequests; +use JSON; + +use constant CONFIG_SVC_TOKEN => 'svc_token_scans'; + +my $query = CGI->new; +my $op = $query->param('op'); +my $token = $query->param('token'); +my $uploads = $query->param('uploads'); + +my $json = JSON->new; +$json->pretty(0); +$json->utf8; + +my $data = {}; +if( !$token || $token ne C4::Context->config(CONFIG_SVC_TOKEN) ) { + $data = { error => 1 }; +} elsif( $op eq 'list' ) { + $data = Koha::ArticleRequests->search_scans_in_process({ list => 1 }); +} elsif( $op eq 'complete' ) { + # uploads should look like: { id1 => 'URL;URL', ... } + $uploads = $json->decode($uploads); + my $cnt = Koha::ArticleRequests->complete_scans($uploads); + $data = { count => $cnt }; +} else { + $data = { error => 2 }; +} +print $query->header( -type => 'application/json', -charset => 'UTF-8' ); +print $json->encode($data); + +=head1 NAME + +opac/svc/scan_requests + +=head1 COPYRIGHT + +Copyright 2018 Rijksmuseum + +=head1 LICENSE + +This file is part of Koha. + +Koha is free software; you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +Koha is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with Koha; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 SYNOPSIS + + GET https://[your Koha server]/cgi-bin/koha/scan_requests?op=list + POST https://[your Koha server]/cgi-bin/koha/scan_requests + (Include op=complete, token=... and uploads=... + +=head1 DESCRIPTION + +This service script (candidate for REST API) allows you to obtain a list of +processing scan requests and allows you to save URLs for processed scans and +complete the requests. + +By using the script you can externalize the matching of open requests and +scanned files, including uploading files to an external web service. Your +solution should just call scan_request at the start with op=list and call +again at the end with op=complete, presenting a token for security and +formatting the results in the described format. + +The uploads parameter should be a JSON representation of a hash structure +like below, where you see request id's and URLs. + + { 1234 => 'https://domain.com/file1', 1235 => 'https://domain.com/file2' } + +Temporarily, the script uses a token to be saved in koha-conf.xml under the +key svc_token_scans. This must be moved later to REST API with appropriate +authorization. + +=head1 AUTHOR + +Marcel de Rooy, Rijksmuseum Amsterdam, The Netherlands + +=cut -- 2.1.4