From a3f92b0bf974e7fa6b251366e51903290189c6e6 Mon Sep 17 00:00:00 2001 From: Andreas Roussos Date: Sun, 8 Mar 2020 19:52:09 +0100 Subject: [PATCH] Bug 24827: Specify UTF-8 as encoding in new_from_xml() The MARC preview of staged records fails when a staged record contains items with UTF-8 characters (see attached screenshots). That's because in EmbedItemsInImportBiblio() the call to MARC::Record->new_from_xml() does not explicitly specify 'UTF-8' as the optional encoding to use. This patch fixes that, and also standardises the encoding name used in direct calls to new_from_xml() to 'UTF-8' instead of 'utf8' or 'utf-8'. Test plan (in a MARC21 instance): 0) Stage for import the test record attached to this bug report (make sure that 'Check for embedded item record data?' is set to 'Yes'), then go to Home > Tools > Manage staged MARC records and click on the filename you've just staged. 1) In the 'Citation' column, click on the hyperlink to launch the MARC preview -- notice how it fails to produce any output apart from the modal heading. 2) Apply the patch, and restart Plack if necessary. 3) Re-launch the MARC preview: this time the output should include the whole record, including the embedded item data in field 952. 4) Run the unit tests to ensure nothing was broken inadvertently: $ prove t $ prove xt Signed-off-by: Owen Leonard --- C4/Biblio.pm | 2 +- C4/ImportBatch.pm | 2 +- Koha/Biblio/Metadata.pm | 2 +- Koha/BiblioUtils/Iterator.pm | 2 +- misc/batchRebuildBiblioTables.pl | 2 +- misc/migration_tools/build_oai_sets.pl | 2 +- opac/opac-readingrecord.pl | 2 +- svc/bib | 2 +- svc/import_bib | 2 +- svc/new_bib | 2 +- t/db_dependent/Koha/Z3950Responder/GenericSession.t | 8 ++++---- t/db_dependent/Koha/Z3950Responder/ZebraSession.t | 4 ++-- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index f745b93d322..60bdc8980a3 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -1158,7 +1158,7 @@ sub GetMarcBiblio { if ($marcxml) { $record = eval { - MARC::Record::new_from_xml( $marcxml, "utf8", + MARC::Record::new_from_xml( $marcxml, "UTF-8", C4::Context->preference('marcflavour') ); }; if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; } diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 11889fd3ee7..eaf8b55c528 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -204,7 +204,7 @@ sub EmbedItemsInImportBiblio { |, { Slice => {} }, $import_record_id ); my @item_fields; for my $import_item ( @$import_items ) { - my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}); + my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}, 'UTF-8'); push @item_fields, $item_marc->field($itemtag); } $record->append_fields(@item_fields); diff --git a/Koha/Biblio/Metadata.pm b/Koha/Biblio/Metadata.pm index bdc1435b5ad..b43a04bdb2f 100644 --- a/Koha/Biblio/Metadata.pm +++ b/Koha/Biblio/Metadata.pm @@ -67,7 +67,7 @@ sub record { my $record; if ( $self->format eq 'marcxml' ) { - $record = eval { MARC::Record::new_from_xml( $self->metadata, 'utf-8', $self->schema ); }; + $record = eval { MARC::Record::new_from_xml( $self->metadata, 'UTF-8', $self->schema ); }; my $marcxml_error = $@; chomp $marcxml_error; unless ($record) { diff --git a/Koha/BiblioUtils/Iterator.pm b/Koha/BiblioUtils/Iterator.pm index 1c8c9d3d870..f8a4e41019d 100644 --- a/Koha/BiblioUtils/Iterator.pm +++ b/Koha/BiblioUtils/Iterator.pm @@ -95,7 +95,7 @@ sub next { return if !$row; my $marcxml = C4::Biblio::GetXmlBiblio( $row->get_column('biblionumber') ); if ( $marcxml ) { - $marc = MARC::Record->new_from_xml( $marcxml ); + $marc = MARC::Record->new_from_xml( $marcxml, 'UTF-8' ); } else { confess "No marcxml column returned in the request."; diff --git a/misc/batchRebuildBiblioTables.pl b/misc/batchRebuildBiblioTables.pl index 15f650377a5..edc7c16d336 100755 --- a/misc/batchRebuildBiblioTables.pl +++ b/misc/batchRebuildBiblioTables.pl @@ -63,7 +63,7 @@ while (my ($biblionumber, $frameworkcode) = $sth->fetchrow) { $marcxml = C4::Charset::StripNonXmlChars( $marcxml ); my $record = eval { - MARC::Record::new_from_xml($marcxml, 'utf8', $marcflavour); + MARC::Record::new_from_xml($marcxml, 'UTF-8', $marcflavour); }; if ($@) { push @errors, $biblionumber; diff --git a/misc/migration_tools/build_oai_sets.pl b/misc/migration_tools/build_oai_sets.pl index a19cc4c025e..64addad2ce8 100755 --- a/misc/migration_tools/build_oai_sets.pl +++ b/misc/migration_tools/build_oai_sets.pl @@ -121,7 +121,7 @@ foreach my $res (@$results) { MARC::File::XML->default_record_format(C4::Context->preference('marcflavour')); my $record; eval { - $record = MARC::Record::new_from_xml($marcxml, "utf8", C4::Context->preference('marcflavour')); + $record = MARC::Record::new_from_xml($marcxml, "UTF-8", C4::Context->preference('marcflavour')); }; if($@) { warn "(biblio $biblionumber) Error while creating record from marcxml: $@"; diff --git a/opac/opac-readingrecord.pl b/opac/opac-readingrecord.pl index 0e7a33a44aa..c13faacaa34 100755 --- a/opac/opac-readingrecord.pl +++ b/opac/opac-readingrecord.pl @@ -97,7 +97,7 @@ foreach my $issue ( @{$issues} ) { if ( $marcxml ) { $marcxml = StripNonXmlChars( $marcxml ); my $marc_rec = - MARC::Record::new_from_xml( $marcxml, 'utf8', + MARC::Record::new_from_xml( $marcxml, 'UTF-8', C4::Context->preference('marcflavour') ); $issue->{normalized_upc} = GetNormalizedUPC( $marc_rec, C4::Context->preference('marcflavour') ); } diff --git a/svc/bib b/svc/bib index 50580e820ee..11cc2609162 100755 --- a/svc/bib +++ b/svc/bib @@ -90,7 +90,7 @@ sub update_bib { my $inxml = $query->param('POSTDATA'); print $query->header(-type => 'text/xml', -charset => 'utf-8'); - my $record = eval {MARC::Record::new_from_xml( $inxml, "utf8", C4::Context->preference('marcflavour'))}; + my $record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", C4::Context->preference('marcflavour'))}; my $do_not_escape = 0; if ($@) { $result->{'status'} = "failed"; diff --git a/svc/import_bib b/svc/import_bib index 8577036ed9f..30734638935 100755 --- a/svc/import_bib +++ b/svc/import_bib @@ -73,7 +73,7 @@ sub import_bib { } my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21'; - my $marc_record = eval {MARC::Record::new_from_xml( $inxml, "utf8", $marcflavour)}; + my $marc_record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", $marcflavour)}; if ($@) { $result->{'status'} = "failed"; $result->{'error'} = $@; diff --git a/svc/new_bib b/svc/new_bib index d9586dd8557..c41d298b2ce 100755 --- a/svc/new_bib +++ b/svc/new_bib @@ -54,7 +54,7 @@ sub add_bib { print $query->header(-type => 'text/xml', -charset => 'utf-8'); my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21'; - my $record = eval {MARC::Record::new_from_xml( $inxml, "utf8", $marcflavour)}; + my $record = eval {MARC::Record::new_from_xml( $inxml, "UTF-8", $marcflavour)}; my $do_not_escape = 0; if ($@) { $result->{'status'} = "failed"; diff --git a/t/db_dependent/Koha/Z3950Responder/GenericSession.t b/t/db_dependent/Koha/Z3950Responder/GenericSession.t index fd52dde5bc7..f888b021218 100644 --- a/t/db_dependent/Koha/Z3950Responder/GenericSession.t +++ b/t/db_dependent/Koha/Z3950Responder/GenericSession.t @@ -109,11 +109,11 @@ subtest 'test_search' => sub { is($rs->size(), 2, 'Two results returned'); - my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw()); + my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw(), 'UTF-8'); ok($returned1, 'Record 1 returned as MARCXML'); is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly'); - my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw()); + my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw(), 'UTF-8'); ok($returned2, 'Record 2 returned as MARCXML'); is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly'); @@ -140,11 +140,11 @@ subtest 'test_search' => sub { my @records = $nodes[0]->getElementsByTagNameNS($marc_ns, 'record'); is(scalar(@records), 2, 'Two results returned'); - $returned1 = MARC::Record->new_from_xml($records[0]->toString()); + $returned1 = MARC::Record->new_from_xml($records[0]->toString(), 'UTF-8'); ok($returned1, 'Record 1 returned as MARCXML'); is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly'); - $returned2= MARC::Record->new_from_xml($records[1]->toString()); + $returned2= MARC::Record->new_from_xml($records[1]->toString(), 'UTF-8'); ok($returned2, 'Record 2 returned as MARCXML'); is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly'); diff --git a/t/db_dependent/Koha/Z3950Responder/ZebraSession.t b/t/db_dependent/Koha/Z3950Responder/ZebraSession.t index 4ba1a605718..3de0d2e8384 100644 --- a/t/db_dependent/Koha/Z3950Responder/ZebraSession.t +++ b/t/db_dependent/Koha/Z3950Responder/ZebraSession.t @@ -106,11 +106,11 @@ subtest 'test_search' => sub { is($rs->size(), 2, 'Two results returned'); - my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw()); + my $returned1 = MARC::Record->new_from_xml($rs->record(0)->raw(), 'UTF-8'); ok ($returned1, 'Record 1 returned as MARCXML'); is($returned1->as_xml, $marc_record_1->as_xml, 'Record 1 returned properly'); - my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw()); + my $returned2= MARC::Record->new_from_xml($rs->record(1)->raw(), 'UTF-8'); ok ($returned2, 'Record 2 returned as MARCXML'); is($returned2->as_xml, $marc_record_2->as_xml, 'Record 2 returned properly'); -- 2.11.0