From be7faa5de8e438c1ffe5ef953a2ab4e586e7fd4e Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Mon, 7 Aug 2017 21:24:48 +0200 Subject: [PATCH] [SIGNED OFF] Bug 19049: Fix regression on stage-marc-import with to_marc plugin Bug 12412 added the use of to_marc plugins allowing arbitrary file formats in stage-marc-import (as long as the plugins can handle them). The feature was not very visible in the code, and when bug 10407 added the marcxml format, it made some changes that broke the use of to_marc. This patch restores the functionality by: [1] Adding a sub RecordsFromMarcPlugin to ImportBatch.pm, specifically addressing the conversion from arbitrary formats to MARC::Record. The original to_marc interface is used: pass it the file contents, and it returns a string consisting of a number of MARC blobs separated by \x1D. Consequently, the call of to_marc is removed from routine BatchStageMarcRecords where it did not belong. The to_marc_plugin parameter is removed and two calls are adjusted accordingly. [2] Instead of a separate combo with plugins, the format combo contains MARC, MARCXML and optionally some plugin formats. [3] The code in stage-marc-import.pl now clearly shows the three main format types: MARC, MARCXML or plugin based. Note: This patch restores more or less the situation after bug 12412, but I would actually recommend to have the to_marc plugins return MARC::Record objects instead of large text strings. In the second example I added a to_marc plugin that actually converts MARC record objects to string format, while RecordsFromMarcPlugin reconverts them to MARC::Records. Test plan: See second patch. Signed-off-by: Marcel de Rooy Signed-off-by: Katrin Fischer --- C4/ImportBatch.pm | 50 +++++++++++++++++----- .../prog/en/modules/tools/stage-marc-import.tt | 20 ++------- misc/stage_file.pl | 2 +- tools/stage-marc-import.pl | 12 ++++-- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index c4550eb..82d6af2 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -352,8 +352,8 @@ sub ModAuthInBatch { ( $batch_id, $num_records, $num_items, @invalid_records ) = BatchStageMarcRecords( - $encoding, $marc_records, - $file_name, $to_marc_plugin, + $record_type, $encoding, + $marc_records, $file_name, $marc_modification_template, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, @@ -367,7 +367,6 @@ sub BatchStageMarcRecords { my $encoding = shift; my $marc_records = shift; my $file_name = shift; - my $to_marc_plugin = shift; my $marc_modification_template = shift; my $comments = shift; my $branch_code = shift; @@ -399,13 +398,6 @@ sub BatchStageMarcRecords { SetImportBatchItemAction($batch_id, 'ignore'); } - $marc_records = Koha::Plugins::Handler->run( - { - class => $to_marc_plugin, - method => 'to_marc', - params => { data => $marc_records } - } - ) if $to_marc_plugin && @$marc_records; my $marc_type = C4::Context->preference('marcflavour'); $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); @@ -1556,6 +1548,44 @@ sub RecordsFromMARCXMLFile { return (\@errors, \@marcRecords); } +=head2 RecordsFromMarcPlugin + + Converts text of input_file into array of MARC records with to_marc plugin + +=cut + +sub RecordsFromMarcPlugin { + my ($input_file, $plugin_class, $encoding) = @_; + + # Read input file + open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; + my ( $text, $marc, @return ); + $/ = "\035"; + while () { + s/^\s+//; + s/\s+$//; + next unless $_; + $text .= $_; + } + close IN; + + # Convert to large MARC blob with plugin + $text = Koha::Plugins::Handler->run({ + class => $plugin_class, + method => 'to_marc', + params => { data => $text }, + }); + + # Convert to array of MARC records + my $marc_type = C4::Context->preference('marcflavour'); + foreach my $blob ( split(/\x1D/, $text) ) { + next if $blob =~ /^\s*$/; + my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding); + push @return, $marcrecord; + } + return \@return; +} + # internal functions sub _create_import_record { diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt index 2383451..4f23ce7 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt @@ -198,27 +198,13 @@ function cbUpload( status, fileid ) { - [% IF plugins %] -
- Transform file to MARC: -
    -
  1. - - -
    2. -
-
- [% END %] - [% IF MarcModificationTemplatesLoop %]
Use MARC Modification Template: diff --git a/misc/stage_file.pl b/misc/stage_file.pl index ad0b726..1f11b6a 100755 --- a/misc/stage_file.pl +++ b/misc/stage_file.pl @@ -116,7 +116,7 @@ sub process_batch { print "... staging MARC records -- please wait\n"; #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible my ($batch_id, $num_valid_records, $num_items, @import_errors) = - BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, undef, $params->{batch_comment}, '', $params->{add_items}, 0, + BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, $params->{batch_comment}, '', $params->{add_items}, 0, 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index 63dd4e5..0ad0128 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -57,8 +57,7 @@ my $item_action = $input->param('item_action'); my $comments = $input->param('comments'); my $record_type = $input->param('record_type'); my $encoding = $input->param('encoding') || 'UTF-8'; -my $format = $input->param('format') || 'ISO2709'; -my $to_marc_plugin = $input->param('to_marc_plugin'); +my $format = $input->param('format'); my $marc_modification_template = $input->param('marc_modification_template_id'); my ( $template, $loggedinuser, $cookie ) = get_template_and_user( @@ -92,8 +91,13 @@ if ($completedJobID) { my ( $errors, $marcrecords ); if( $format eq 'MARCXML' ) { ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromMARCXMLFile( $file, $encoding); - } else { + } elsif( $format eq 'MARC' ) { ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromISO2709File( $file, $record_type, $encoding ); + } elsif( $format ) { # plugin + $errors = []; + $marcrecords = C4::ImportBatch::RecordsFromMarcPlugin( $file, $format, $encoding ); + } else { + die "No format specified"; } warn "$filename: " . ( join ',', @$errors ) if @$errors; # no need to exit if we have no records (or only errors) here @@ -141,7 +145,7 @@ if ($completedJobID) { BatchStageMarcRecords( $record_type, $encoding, $marcrecords, $filename, - $to_marc_plugin, $marc_modification_template, + $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback( $job, $dbh ) -- 2.7.4