From be7faa5de8e438c1ffe5ef953a2ab4e586e7fd4e Mon Sep 17 00:00:00 2001
From: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
Date: Mon, 7 Aug 2017 21:24:48 +0200
Subject: [PATCH] [SIGNED OFF] Bug 19049: Fix regression on stage-marc-import
 with to_marc plugin

Bug 12412 added the use of to_marc plugins allowing arbitrary file formats
in stage-marc-import (as long as the plugins can handle them). The feature
was not very visible in the code, and when bug 10407 added the marcxml
format, it made some changes that broke the use of to_marc.

This patch restores the functionality by:
[1] Adding a sub RecordsFromMarcPlugin to ImportBatch.pm, specifically
    addressing the conversion from arbitrary formats to MARC::Record.
    The original to_marc interface is used: pass it the file contents,
    and it returns a string consisting of a number of MARC blobs separated
    by \x1D.
    Consequently, the call of to_marc is removed from routine
    BatchStageMarcRecords where it did not belong. The to_marc_plugin
    parameter is removed and two calls are adjusted accordingly.
[2] Instead of a separate combo with plugins, the format combo contains
    MARC, MARCXML and optionally some plugin formats.
[3] The code in stage-marc-import.pl now clearly shows the three main
    format types: MARC, MARCXML or plugin based.

Note: This patch restores more or less the situation after bug 12412, but
I would actually recommend to have the to_marc plugins return MARC::Record
objects instead of large text strings. In the second example I added a
to_marc plugin that actually converts MARC record objects to string format,
while RecordsFromMarcPlugin reconverts them to MARC::Records.

Test plan:
See second patch.

Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>

Signed-off-by: Katrin Fischer <katrin.fischer.83@web.de>
---
 C4/ImportBatch.pm                                  | 50 +++++++++++++++++-----
 .../prog/en/modules/tools/stage-marc-import.tt     | 20 ++-------
 misc/stage_file.pl                                 |  2 +-
 tools/stage-marc-import.pl                         | 12 ++++--
 4 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm
index c4550eb..82d6af2 100644
--- a/C4/ImportBatch.pm
+++ b/C4/ImportBatch.pm
@@ -352,8 +352,8 @@ sub ModAuthInBatch {
 
 ( $batch_id, $num_records, $num_items, @invalid_records ) =
   BatchStageMarcRecords(
-    $encoding,                   $marc_records,
-    $file_name,                  $to_marc_plugin,
+    $record_type,                $encoding,
+    $marc_records,               $file_name,
     $marc_modification_template, $comments,
     $branch_code,                $parse_items,
     $leave_as_staging,           $progress_interval,
@@ -367,7 +367,6 @@ sub BatchStageMarcRecords {
     my $encoding = shift;
     my $marc_records = shift;
     my $file_name = shift;
-    my $to_marc_plugin = shift;
     my $marc_modification_template = shift;
     my $comments = shift;
     my $branch_code = shift;
@@ -399,13 +398,6 @@ sub BatchStageMarcRecords {
         SetImportBatchItemAction($batch_id, 'ignore');
     }
 
-    $marc_records = Koha::Plugins::Handler->run(
-        {
-            class  => $to_marc_plugin,
-            method => 'to_marc',
-            params => { data => $marc_records }
-        }
-    ) if $to_marc_plugin && @$marc_records;
 
     my $marc_type = C4::Context->preference('marcflavour');
     $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
@@ -1556,6 +1548,44 @@ sub RecordsFromMARCXMLFile {
     return (\@errors, \@marcRecords);
 }
 
+=head2 RecordsFromMarcPlugin
+
+    Converts text of input_file into array of MARC records with to_marc plugin
+
+=cut
+
+sub RecordsFromMarcPlugin {
+    my ($input_file, $plugin_class, $encoding) = @_;
+
+    # Read input file
+    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
+    my ( $text, $marc, @return );
+    $/ = "\035";
+    while (<IN>) {
+        s/^\s+//;
+        s/\s+$//;
+        next unless $_;
+        $text .= $_;
+    }
+    close IN;
+
+    # Convert to large MARC blob with plugin
+    $text = Koha::Plugins::Handler->run({
+        class  => $plugin_class,
+        method => 'to_marc',
+        params => { data => $text },
+    });
+
+    # Convert to array of MARC records
+    my $marc_type = C4::Context->preference('marcflavour');
+    foreach my $blob ( split(/\x1D/, $text) ) {
+        next if $blob =~ /^\s*$/;
+        my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding);
+        push @return, $marcrecord;
+    }
+    return \@return;
+}
+
 # internal functions
 
 sub _create_import_record {
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt
index 2383451..4f23ce7 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt
@@ -198,27 +198,13 @@ function cbUpload( status, fileid ) {
         <select name='format' id='format'>
             <option value='ISO2709'>MARC</option>
             <option value='MARCXML'>MARCXML</option>
+            [% FOREACH p IN plugins %]
+                <option value="[% p.metadata.class %]">Other formats, handled by plugin [% p.metadata.name %]</option>
+            [% END %]
         </select>
     </li>
 </ol></fieldset>
 
-  [% IF plugins %]
-    <fieldset class="rows">
-      <legend>Transform file to MARC:</legend>
-      <ol>
-        <li>
-          <label for="comments">Convert file to MARC using the following plugin: </label>
-          <select name="to_marc_plugin" id="to_marc_plugin">
-            <option value="">Do not use.</option>
-              [% FOREACH p IN plugins %]
-                <option value="[% p.metadata.class %]">[% p.metadata.name %]</option>
-              [% END %]
-          </select>
-        </li>
-      </ol>
-    </fieldset>
-  [% END %]
-
   [% IF MarcModificationTemplatesLoop %]
     <fieldset class="rows">
       <legend>Use MARC Modification Template:</legend>
diff --git a/misc/stage_file.pl b/misc/stage_file.pl
index ad0b726..1f11b6a 100755
--- a/misc/stage_file.pl
+++ b/misc/stage_file.pl
@@ -116,7 +116,7 @@ sub process_batch {
     print "... staging MARC records -- please wait\n";
     #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
     my ($batch_id, $num_valid_records, $num_items, @import_errors) =
-        BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, undef, $params->{batch_comment}, '', $params->{add_items}, 0,
+        BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, $params->{batch_comment}, '', $params->{add_items}, 0,
                               100, \&print_progress_and_commit);
     print "... finished staging MARC records\n";
 
diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl
index 63dd4e5..0ad0128 100755
--- a/tools/stage-marc-import.pl
+++ b/tools/stage-marc-import.pl
@@ -57,8 +57,7 @@ my $item_action                = $input->param('item_action');
 my $comments                   = $input->param('comments');
 my $record_type                = $input->param('record_type');
 my $encoding                   = $input->param('encoding') || 'UTF-8';
-my $format                     = $input->param('format') || 'ISO2709';
-my $to_marc_plugin             = $input->param('to_marc_plugin');
+my $format                     = $input->param('format');
 my $marc_modification_template = $input->param('marc_modification_template_id');
 
 my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
@@ -92,8 +91,13 @@ if ($completedJobID) {
     my ( $errors, $marcrecords );
     if( $format eq 'MARCXML' ) {
         ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromMARCXMLFile( $file, $encoding);
-    } else {
+    } elsif( $format eq 'MARC' ) {
         ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromISO2709File( $file, $record_type, $encoding );
+    } elsif( $format ) { # plugin
+        $errors = [];
+        $marcrecords = C4::ImportBatch::RecordsFromMarcPlugin( $file, $format, $encoding );
+    } else {
+        die "No format specified";
     }
     warn "$filename: " . ( join ',', @$errors ) if @$errors;
         # no need to exit if we have no records (or only errors) here
@@ -141,7 +145,7 @@ if ($completedJobID) {
       BatchStageMarcRecords(
         $record_type,    $encoding,
         $marcrecords,    $filename,
-        $to_marc_plugin, $marc_modification_template,
+        $marc_modification_template,
         $comments,       '',
         $parse_items,    0,
         50, staging_progress_callback( $job, $dbh )
-- 
2.7.4