From 28be579f0be24e71903042864dfd8934c18c9080 Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Thu, 30 Jul 2015 19:19:44 +0200 Subject: [PATCH] Bug 14321: Using Koha::Upload in tools/upload-file.pl Content-Type: text/plain; charset=utf-8 This patch introduces Koha::Upload. It will replace the modules C4::UploadedFile.pm and the new C4::UploadedFiles.pm (from BZ 6874). This operation will be performed in a few steps. As a first step, we will use Koha::Upload instead of UploadedFile in the following places. (In these three places files are uploaded temporarily.) [1] stage-marc-import.pl [2] upload-cover-image.pl [3] offline_circ/process_koc.pl and enqueue_koc.pl A new file-upload.js replaces file-upload.inc in the associated template. We now use ajax to get progress figures instead of launching perl script upload-file-progress. Also we do not need the jquery plugin ajaxfileupload.js anymore. A subsequent patch will remove these files. The js changes now also allow for aborting a file upload. This feature has been added to stage-marc-import only for now. NOTE: In editing the process_koc.tt I noticed that the form enqueuefile was hidden and no longer used (with associated code in process_koc.pl). When a file has been uploaded, I display the form again (with the Apply directly button). The code still works. Test plan: [1] Upload a marc file in stage-marc-import. [2] Check new entry in table uploaded_files. Look for the file in the directory defined by upload_path in koha-conf.xml. [3] Upload another (larger) file and abort the upload. Check table and directory again. [4] Verify that Stage for import still works as expected. [5] Test Upload local cover image. (Enable OPACLocalCoverImages.) You can test an individual image or a zip file including images and a file called datalink.txt (with lines biblionumber,filename). [6] Test uploading a offline circulation file: Enable AllowOfflineCirculation, and create a koc file (plain text): Line1: Version=1.0\tA=1\tB=2 Line2: 2015-08-06 08:00:00 345\treturn\t[barcode] Note: Replace tabs and barcode. The number of tabs is essential! Checkout the item with your barcode. Go to Offline circulation file upload. Upload and click Apply directly. Checkout again. Upload again, click Add to offline circulation queue. --- Koha/Upload.pm | 272 ++++++++++++++++++++ koha-tmpl/intranet-tmpl/prog/en/js/file-upload.js | 32 +++ .../prog/en/modules/offline_circ/process_koc.tt | 39 ++- .../prog/en/modules/tools/stage-marc-import.tt | 64 ++++- .../prog/en/modules/tools/upload-images.tt | 45 +++- offline_circ/enqueue_koc.pl | 6 +- offline_circ/process_koc.pl | 8 +- tools/stage-marc-import.pl | 9 +- tools/upload-cover-image.pl | 8 +- tools/upload-file.pl | 55 ++-- 10 files changed, 464 insertions(+), 74 deletions(-) create mode 100644 Koha/Upload.pm create mode 100644 koha-tmpl/intranet-tmpl/prog/en/js/file-upload.js diff --git a/Koha/Upload.pm b/Koha/Upload.pm new file mode 100644 index 0000000..c3a7e47 --- /dev/null +++ b/Koha/Upload.pm @@ -0,0 +1,272 @@ +package Koha::Upload; + +# Copyright 2015 Rijksmuseum +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +Koha::Upload - Facilitate file upload + +=head1 SYNOPSIS + + use Koha::Upload; + +=head1 DESCRIPTION + + This class + +=head1 METHODS + +=head2 new + + Create object (via Class::Accessor). + +=head1 PROPERTIES + +=head2 ??? + + ??? + +=head1 ADDITIONAL COMMENTS + +=cut + +use constant KOHA_UPLOAD => 'koha_upload'; +use constant BYTES_DIGEST => 2048; + +use Modern::Perl; +use CGI; # no utf8 flag, since it may interfere with binary uploads +use Digest::MD5; +use File::Spec; +use IO::File; +use Time::HiRes; + +use base qw(Class::Accessor); + +use C4::Context; + +__PACKAGE__->mk_ro_accessors( qw| name +|); + +=head2 new + + Returns new object based on Class::Accessor. + +=cut + +sub new { + my ( $class, $params ) = @_; + my $self = $class->SUPER::new(); + $self->_init( $params ); + return $self; +} + +=head2 cgi + + Returns new object based on Class::Accessor. + +=cut + +sub cgi { + my ( $self ) = @_; + + # Next call handles the actual upload via CGI hook. + # The third parameter (0) below means: no CGI temporary storage. + # Cancelling an upload will make CGI abort the script; no problem, + # the file(s) without db entry will be removed later. + my $query = CGI::->new( sub { $self->_hook(@_); }, {}, 0 ); + if( $query ) { + $self->_done; + return $query; + } +} + +=head2 result + + Returns new object based on Class::Accessor. + +=cut + +sub result { + my ( $self ) = @_; + my @a = map { $self->{files}->{$_}->{id} } keys $self->{files}; + return join ',', @a; +} + +=head2 get + + Returns array + optional parameter: filehandle => 1 (name, path only by default) + +=cut + +sub get { + my ( $self, $params ) = @_; + my $temp= $self->_lookup( $params ); + my ( @rv, $res ); + foreach my $r ( @$temp ) { + $res->{name} = $r->{filename}; + $res->{path}= $self->_full_fname($r); + $res->{fh} = IO::File->new( $res->{path}, "r" ) + if $params->{filehandle}; + push @rv, $res; + last if !wantarray; + } + return wantarray? @rv: $res; +} + +sub DESTROY { +} + +# ************** INTERNAL ROUTINES ******************************************** + +sub _init { + my ( $self, $params ) = @_; + + $self->{rootdir} = C4::Context->config('upload_path'); + $self->{tmpdir} = File::Spec->tmpdir; + if( exists $params->{tmp} || exists $params->{temp} || + !$params->{category} ) { + $self->{temporary} = 1; + $self->{category} = undef; + } else { + $self->{category} = $params->{category}; + } + + $self->{errors} = []; + $self->{files} = {}; + $self->{uid} = C4::Context->userenv->{number} if C4::Context->userenv; +} + +sub _fh { + my ( $self, $filename ) = @_; + if( $self->{files}->{$filename} ) { + return $self->{files}->{$filename}->{fh}; + } +} + +sub _create_file { + my ( $self, $filename ) = @_; + my $fh; + if( $self->{files}->{$filename} && + $self->{files}->{$filename}->{errcode} ) { + #skip + } elsif( $self->{temporary} && !$self->{tmpdir} ) { + $self->{files}->{$filename}->{errcode} = 2; + } else { + my $dir = $self->_dir; + my $fn = $self->{files}->{$filename}->{hash}. '_'. $filename; + $fh = IO::File->new( "$dir/$fn", "w"); + if( $fh ) { + $fh->binmode; + $self->{files}->{$filename}->{fh}= $fh; + } else { + $self->{files}->{$filename}->{errcode} = 1; + # push @{$self->{errors}}, [ 1, $filename ]; + } + } + return $fh; +} + +sub _dir { + my ( $self ) = @_; + my $dir = $self->{temporary}? $self->{tmpdir}: $self->{rootdir}; + $dir.= '/'. ( $self->{category}? $self->{category}: KOHA_UPLOAD ); + mkdir $dir if !-d $dir; + return $dir; +} + +sub _full_fname { + my ( $self, $rec ) = @_; + if( ref $rec ) { + return ( $rec->{category}? $self->{rootdir}: $self->{tmpdir} ). + '/'. $rec->{dir}. '/'. $rec->{hashvalue}. '_'. $rec->{filename}; + } +} + +sub _hook { + my ( $self, $filename, $buffer, $bytes_read, $data ) = @_; + $self->_compute( $filename, $buffer ); + my $fh = $self->_fh( $filename ) // $self->_create_file( $filename ); + print $fh $buffer if $fh; +} + +sub _done { + my ( $self ) = @_; + $self->{done} = 1; + foreach my $f ( keys $self->{files} ) { + my $fh = $self->_fh($f); + $self->_register( $f, $fh? tell( $fh ): undef ) + if !$self->{files}->{$f}->{errcode}; + $fh->close if $fh; + } +} + +sub _register { + my ( $self, $filename, $size ) = @_; + my $dbh= C4::Context->dbh; + my $sql= "INSERT INTO uploaded_files (hashvalue, filename, dir, filesize, + owner, categorycode) VALUES (?,?,?,?,?,?)"; + my @pars= ( $self->{files}->{$filename}->{hash}, + $filename, + $self->{temporary}? KOHA_UPLOAD: $self->{category}, + $size, + $self->{uid}, + $self->{category}, + ); + $dbh->do( $sql, undef, @pars ); + my $i = $dbh->last_insert_id(undef, undef, 'uploaded_files', undef); + $self->{files}->{$filename}->{id} = $i if $i; +} + +sub _lookup { + my ( $self, $params ) = @_; + my $dbh = C4::Context->dbh; + my $sql = 'SELECT id,hashvalue,filename,dir,categorycode '. + 'FROM uploaded_files '; + if( $params->{id} ) { + $sql.= "WHERE id=?"; + } else { + $sql.= "WHERE hashvalue=?"; + } + my $temp= $dbh->selectall_arrayref( $sql, { Slice => {} }, + ( $params->{id} // $params->{hashvalue} // 0 ) ); + return $temp; +} + +sub _compute { +# Computes hash value when sub hook feeds the first block +# For temporary files, the id is made unique with time + my ( $self, $name, $block ) = @_; + if( !$self->{files}->{$name}->{hash} ) { + my $str = $name. ( $self->{uid} // '0' ). + ( $self->{temporary}? Time::HiRes::time(): '' ). + ( $self->{category} // 'tmp' ). substr( $block, 0, BYTES_DIGEST ); + # since Digest cannot handle wide chars, we need to encode here + # there could be a wide char in the filename or the category + my $h = Digest::MD5::md5_hex( Encode::encode_utf8( $str ) ); + $self->{files}->{$name}->{hash} = $h; + } +} + +=head1 AUTHOR + + Marcel de Rooy, Rijksmuseum Amsterdam, The Netherlands + +=cut + +1; diff --git a/koha-tmpl/intranet-tmpl/prog/en/js/file-upload.js b/koha-tmpl/intranet-tmpl/prog/en/js/file-upload.js new file mode 100644 index 0000000..789f11d --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/js/file-upload.js @@ -0,0 +1,32 @@ +function AjaxUpload ( input, progressbar, callback ) { + // input and progressbar are jQuery objects + // callback is the callback function for completion + var formData= new FormData(); + $.each( input.prop('files'), function( dx, file ) { + formData.append( "uploadfile", file ); + }); + var xhr= new XMLHttpRequest(); + var url= '/cgi-bin/koha/tools/upload-file.pl'; + progressbar.val( 0 ); + progressbar.next('.fileuploadpercent').text( '0' ); + xhr.open('POST', url, true); + xhr.upload.onprogress = function (e) { + var p = Math.round( (e.loaded/e.total) * 100 ); + progressbar.val( p ); + progressbar.next('.fileuploadpercent').text( p ); + } + xhr.onload = function (e) { + var data = JSON.parse( xhr.responseText ); + if( data.status == 'done' ) { + progressbar.val( 100 ); + progressbar.next('.fileuploadpercent').text( '100' ); + } + callback( data.status, data.fileid ); + } + xhr.onerror = function (e) { + // Probably only fires for network failure + alert('An error occurred while uploading.'); + } + xhr.send( formData ); + return xhr; +} diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/offline_circ/process_koc.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/offline_circ/process_koc.tt index 76c317a..2a44d82 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/offline_circ/process_koc.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/offline_circ/process_koc.tt @@ -1,22 +1,43 @@ [% INCLUDE 'doc-head-open.inc' %] Koha › Circulation › Offline circulation file upload [% INCLUDE 'doc-head-close.inc' %] -[% INCLUDE 'file-upload.inc' %] + + + + + #fileuploadprogress,#jobprogress { width:150px;height:10px;border:1px solid #666;background:url('[% interface %]/[% theme %]/img/progress.png') -300px 0px no-repeat; } + + @@ -90,11 +131,20 @@ function CheckForm(f) { -
+
+ + +
-
Upload progress:
0%
-
+
+
Upload progress: + + + 0% +
+
+
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/upload-images.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/upload-images.tt index 4565995..812df23 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/upload-images.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/upload-images.tt @@ -1,14 +1,40 @@ [% INCLUDE 'doc-head-open.inc' %] Koha › Tools › Upload images [% INCLUDE 'doc-head-close.inc' %] -[% INCLUDE 'file-upload.inc' %] + + + + #fileuploadprogress,#jobprogress { width:150px;height:10px;border:1px solid #666;background:url('[% interface %]/[% theme %]/img/progress.png') -300px 0px no-repeat; } + +