Attachment 100320 Details for Bug 24827 – Bug 24827: Specify UTF-8 as encoding in new_from_xml()

[patch] Bug 24827: Specify UTF-8 as encoding in new_from_xml()

Bug-24827-Specify-UTF-8-as-encoding-in-newfromxml.patch (text/plain), 10.18 KB, created by Andreas Roussos on 2020-03-08 22:12:48 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Andreas Roussos

Created: 2020-03-08 22:12:48 UTC

Size: 10.18 KB

patch

obsolete

>From bcad1e93892bf4eee840696cf9dddaeb747723ca Mon Sep 17 00:00:00 2001
>From: Andreas Roussos <a.roussos@dataly.gr>
>Date: Sun, 8 Mar 2020 19:52:09 +0100
>Subject: [PATCH] Bug 24827: Specify UTF-8 as encoding in new_from_xml()
>
>The MARC preview of staged records fails when a staged record contains
>items with UTF-8 characters (see attached screenshots). That's because
>in EmbedItemsInImportBiblio() the call to MARC::Record->new_from_xml()
>does not explicitly specify 'UTF-8' as the optional encoding to use.
>
>This patch fixes that, and also standardises the encoding name used in
>direct calls to new_from_xml() to 'UTF-8' instead of 'utf8' or 'utf-8'.
>
>Test plan (in a MARC21 instance):
>0) Stage for import the test record attached to this bug report (make
>   sure that 'Check for embedded item record data?' is set to 'Yes'),
>   then go to Home > Tools > Manage staged MARC records and click on
>   the filename you've just staged.
>1) In the 'Citation' column, click on the hyperlink to launch the MARC
>   preview -- notice how it fails to produce any output apart from the
>   modal heading.
>2) Apply the patch, and restart Plack if necessary.
>3) Re-launch the MARC preview: this time the output should include the
>   whole record, including the embedded item data in field 952.
>4) Run the unit tests to ensure nothing was broken inadvertently:
>   $ prove t
>   $ prove xt
>---
> C4/Biblio.pm                                        | 2 +-
> C4/ImportBatch.pm                                   | 2 +-
> Koha/Biblio/Metadata.pm                             | 2 +-
> Koha/BiblioUtils/Iterator.pm                        | 2 +-
> misc/batchRebuildBiblioTables.pl                    | 2 +-
> misc/migration_tools/build_oai_sets.pl              | 2 +-
> opac/opac-readingrecord.pl                          | 2 +-
> svc/bib                                             | 2 +-
> svc/import_bib                                      | 2 +-
> svc/new_bib                                         | 2 +-
> t/db_dependent/Koha/Z3950Responder/GenericSession.t | 8 ++++----
> t/db_dependent/Koha/Z3950Responder/ZebraSession.t   | 4 ++--
> 12 files changed, 16 insertions(+), 16 deletions(-)
>
>diff --git a/C4/Biblio.pm b/C4/Biblio.pm
>index f745b93d32..60bdc8980a 100644
>--- a/C4/Biblio.pm
>+++ b/C4/Biblio.pm
>@@ -1158,7 +1158,7 @@ sub GetMarcBiblio {
> 
>     if ($marcxml) {
>         $record = eval {
>-            MARC::Record::new_from_xml( $marcxml, "utf8",
>+            MARC::Record::new_from_xml( $marcxml, "UTF-8",
>                 C4::Context->preference('marcflavour') );
>         };
>         if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; }
>diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm
>index 11889fd3ee..eaf8b55c52 100644
>--- a/C4/ImportBatch.pm
>+++ b/C4/ImportBatch.pm
>@@ -204,7 +204,7 @@ sub EmbedItemsInImportBiblio {
>     |, { Slice => {} }, $import_record_id );
>     my @item_fields;
>     for my $import_item ( @$import_items ) {
>-        my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml});
>+        my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}, 'UTF-8');
>         push @item_fields, $item_marc->field($itemtag);
>     }
>     $record->append_fields(@item_fields);
>diff --git a/Koha/Biblio/Metadata.pm b/Koha/Biblio/Metadata.pm
>index bdc1435b5a..b43a04bdb2 100644
>--- a/Koha/Biblio/Metadata.pm
>+++ b/Koha/Biblio/Metadata.pm
>@@ -67,7 +67,7 @@ sub record {
>     my $record;
> 
>     if ( $self->format eq 'marcxml' ) {
>-        $record = eval { MARC::Record::new_from_xml( $self->metadata, 'utf-8', $self->schema ); };
>+        $record = eval { MARC::Record::new_from_xml( $self->metadata, 'UTF-8', $self->schema ); };
>         my $marcxml_error = $@;
>         chomp $marcxml_error;
>         unless ($record) {
>diff --git a/Koha/BiblioUtils/Iterator.pm b/Koha/BiblioUtils/Iterator.pm
>index 1c8c9d3d87..f8a4e41019 100644
>--- a/Koha/BiblioUtils/Iterator.pm
>+++ b/Koha/BiblioUtils/Iterator.pm
>@@ -95,7 +95,7 @@ sub next {
>     return if !$row;
>     my $marcxml = C4::Biblio::GetXmlBiblio( $row->get_column('biblionumber') );
>     if ( $marcxml ) {
>-        $marc = MARC::Record->new_from_xml( $marcxml );
>+        $marc = MARC::Record->new_from_xml( $marcxml, 'UTF-8' );
>     }
>     else {
>         confess "No marcxml column returned in the request.";
>diff --git a/misc/batchRebuildBiblioTables.pl b/misc/batchRebuildBiblioTables.pl
>index 15f650377a..edc7c16d33 100755
>--- a/misc/batchRebuildBiblioTables.pl
>+++ b/misc/batchRebuildBiblioTables.pl
>@@ -63,7 +63,7 @@ while (my ($biblionumber, $frameworkcode) = $sth->fetchrow) {
> 
>     $marcxml = C4::Charset::StripNonXmlChars( $marcxml );
>     my $record = eval {
>-        MARC::Record::new_from_xml($marcxml, 'utf8', $marcflavour);
>+        MARC::Record::new_from_xml($marcxml, 'UTF-8', $marcflavour);
>     };
>     if ($@) {
>         push @errors, $biblionumber;
>diff --git a/misc/migration_tools/build_oai_sets.pl b/misc/migration_tools/build_oai_sets.pl
>index a19cc4c025..64addad2ce 100755
>--- a/misc/migration_tools/build_oai_sets.pl
>+++ b/misc/migration_tools/build_oai_sets.pl
>@@ -121,7 +121,7 @@ foreach my $res (@$results) {
>     MARC::File::XML->default_record_format(C4::Context->preference('marcflavour'));
>     my $record;
>     eval {
>-        $record = MARC::Record::new_from_xml($marcxml, "utf8", C4::Context->preference('marcflavour'));
>+        $record = MARC::Record::new_from_xml($marcxml, "UTF-8", C4::Context->preference('marcflavour'));
>     };
>     if($@) {
>         warn "(biblio $biblionumber) Error while creating record from marcxml: $@";
>diff --git a/opac/opac-readingrecord.pl b/opac/opac-readingrecord.pl
>index 0e7a33a44a..c13faacaa3 100755
>--- a/opac/opac-readingrecord.pl
>+++ b/opac/opac-readingrecord.pl
>@@ -97,7 +97,7 @@ foreach my $issue ( @{$issues} ) {
>     if ( $marcxml ) {
>         $marcxml = StripNonXmlChars( $marcxml );
>         my $marc_rec =
>-          MARC::Record::new_from_xml( $marcxml, 'utf8',
>+          MARC::Record::new_from_xml( $marcxml, 'UTF-8',
>             C4::Context->preference('marcflavour') );
>         $issue->{normalized_upc} = GetNormalizedUPC( $marc_rec, C4::Context->preference('marcflavour') );
>     }
>diff --git a/svc/bib b/svc/bib
>index 50580e820e..11cc260916 100755
>--- a/svc/bib
>+++ b/svc/bib
>@@ -90,7 +90,7 @@ sub update_bib {
>     my $inxml = $query->param('POSTDATA');
>     print $query->header(-type => 'text/xml', -charset => 'utf-8');
> 
>-    my $record = eval {MARC::Record::new_from_xml( $inxml, "utf8", C4::Context->preference('marcflavour'))};
>+    my $record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", C4::Context->preference('marcflavour'))};
>     my $do_not_escape = 0;
>     if ($@) {
>         $result->{'status'} = "failed";
>diff --git a/svc/import_bib b/svc/import_bib
>index 8577036ed9..3073463893 100755
>--- a/svc/import_bib
>+++ b/svc/import_bib
>@@ -73,7 +73,7 @@ sub import_bib {
>     }
> 
>     my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
>-    my $marc_record = eval {MARC::Record::new_from_xml( $inxml, "utf8", $marcflavour)};
>+    my $marc_record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", $marcflavour)};
>     if ($@) {
>         $result->{'status'} = "failed";
>         $result->{'error'} = $@;
>diff --git a/svc/new_bib b/svc/new_bib
>index d9586dd855..c41d298b2c 100755
>--- a/svc/new_bib
>+++ b/svc/new_bib
>@@ -54,7 +54,7 @@ sub add_bib {
>     print $query->header(-type => 'text/xml', -charset => 'utf-8');
> 
>     my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
>-    my $record = eval {MARC::Record::new_from_xml( $inxml, "utf8", $marcflavour)};
>+    my $record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", $marcflavour)};
>     my $do_not_escape = 0;
>     if ($@) {
>         $result->{'status'} = "failed";
>diff --git a/t/db_dependent/Koha/Z3950Responder/GenericSession.t b/t/db_dependent/Koha/Z3950Responder/GenericSession.t
>index fd52dde5bc..f888b02121 100644
>--- a/t/db_dependent/Koha/Z3950Responder/GenericSession.t
>+++ b/t/db_dependent/Koha/Z3950Responder/GenericSession.t
>@@ -109,11 +109,11 @@ subtest 'test_search' => sub {
> 
>     is($rs->size(), 2, 'Two results returned');
> 
>-    my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw());
>+    my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw(), 'UTF-8');
>     ok($returned1, 'Record 1 returned as MARCXML');
>     is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly');
> 
>-    my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw());
>+    my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw(), 'UTF-8');
>     ok($returned2, 'Record 2 returned as MARCXML');
>     is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly');
> 
>@@ -140,11 +140,11 @@ subtest 'test_search' => sub {
>     my @records = $nodes[0]->getElementsByTagNameNS($marc_ns, 'record');
>     is(scalar(@records), 2, 'Two results returned');
> 
>-    $returned1 = MARC::Record->new_from_xml($records[0]->toString());
>+    $returned1 = MARC::Record->new_from_xml($records[0]->toString(), 'UTF-8');
>     ok($returned1, 'Record 1 returned as MARCXML');
>     is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly');
> 
>-    $returned2= MARC::Record->new_from_xml($records[1]->toString());
>+    $returned2= MARC::Record->new_from_xml($records[1]->toString(), 'UTF-8');
>     ok($returned2, 'Record 2 returned as MARCXML');
>     is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly');
> 
>diff --git a/t/db_dependent/Koha/Z3950Responder/ZebraSession.t b/t/db_dependent/Koha/Z3950Responder/ZebraSession.t
>index 4ba1a60571..3de0d2e838 100644
>--- a/t/db_dependent/Koha/Z3950Responder/ZebraSession.t
>+++ b/t/db_dependent/Koha/Z3950Responder/ZebraSession.t
>@@ -106,11 +106,11 @@ subtest 'test_search' => sub {
> 
>     is($rs->size(), 2, 'Two results returned');
> 
>-    my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw());
>+    my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw(), 'UTF-8');
>     ok ($returned1, 'Record 1 returned as MARCXML');
>     is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly');
> 
>-    my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw());
>+    my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw(), 'UTF-8');
>     ok ($returned2, 'Record 2 returned as MARCXML');
>     is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly');
> 
>-- 
>2.11.0

Actions: View | Diff | Splinter Review

Attachments on bug 24827: 100313 | 100314 | 100315 | 100320 | 100410 | 100694 | 100695 | 100737 | 100738 | 100739