From 03953de1cb977b4fff9ead6a88ca98e14614b6be Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Wed, 27 Oct 2021 12:00:23 +0200 Subject: [PATCH] Bug 29333: Fix encoding of imported UNIMARC authorities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MARC::Record and MARC::File::* modules sometimes use the position 09 of the leader to detect encoding. A blank character means 'MARC-8' while an 'a' means 'UTF-8'. In a UNIMARC authority this position is used to store the authority type (see https://www.transition-bibliographique.fr/wp-content/uploads/2021/02/AIntroLabel-2004.pdf [FR]). In this case, 'a' means 'Personal Name'. The result is that the import will succeed for a Personal Name authority, but it will fail for all other authority types. Steps to reproduce: 0. Be sure to have a Koha UNIMARC instance. 1. Download the MARCXML for "Honoré de Balzac" curl -o balzac.marcxml https://www.idref.fr/02670305X.xml 2. Verify that it's encoded in UTF-8 file balzac.marcxml (should output "balzac.marcxml: XML 1.0 document, UTF-8 Unicode text") 3. Go to Tools » Stage MARC for import and import balzac.marcxml with the following settings: Record type: Authority Character encoding: UTF-8 Format: MARCXML Do not touch the other settings 4. Once imported, go to the staged MARC management tool and find your batch. Click on the authority title "Balzac Honoré de 1799-1850" to show the MARC inside a modal window. There should be no encoding issue. 5. Write down the imported record id (the number in column '#') and go to the MARC authority editor. Replace all URL parameters by 'breedingid=THE_ID_YOU_WROTE_DOWN' The URL should look like this: /cgi-bin/koha/authorities/authorities.pl?breedingid=198 You should see no encoding issues. Do not save the record. 6. Import the batch into the catalog. Verify that the authority record has no encoding issue. 7. Now download the MARCXML for "Athènes (Grèce)" curl -o athènes.marcxml https://www.idref.fr/027290530.xml 8. Repeat steps 2 to 6 using athènes.marcxml file. At steps 4 and 5 you should see encoding issues and that the position 9 of the leader was rewritten from 'c' to 'a'. Strangely, importing this batch fix the encoding issue, but we still lose the information in position 09 of the leader This patch makes use of the MARCXML representation of the record instead of the ISO2709 representation, because, unlike MARC::Record::new_from_usmarc, MARC::Record::new_from_xml allows us to pass directly the encoding and the format, which prevents data to be double encoded when position 09 of the leader is different that 'a' Test plan: - Follow the "steps to reproduce" above and verify that you have no encoding issues. --- C4/ImportBatch.pm | 19 +++++++++++-------- authorities/authorities.pl | 10 ++++++---- catalogue/showmarc.pl | 4 +++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index db26883354..8935d0ed04 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -195,8 +195,10 @@ sub GetImportRecordMarc { sub GetRecordFromImportBiblio { my ( $import_record_id, $embed_items ) = @_; - my ($marc) = GetImportRecordMarc($import_record_id); - my $record = MARC::Record->new_from_usmarc($marc); + my ($xml, $encoding) = GetImportRecordMarcXML($import_record_id); + my $marcflavour = C4::Context->preference('marcflavour'); + my $format = $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'; + my $record = MARC::Record->new_from_xml($xml, $encoding, $format); EmbedItemsInImportBiblio( $record, $import_record_id ) if $embed_items; @@ -223,7 +225,7 @@ sub EmbedItemsInImportBiblio { =head2 GetImportRecordMarcXML - my $marcxml = GetImportRecordMarcXML($import_record_id); + my ($marcxml, $encoding) = GetImportRecordMarcXML($import_record_id); =cut @@ -231,12 +233,13 @@ sub GetImportRecordMarcXML { my ($import_record_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?"); - $sth->execute($import_record_id); - my ($marcxml) = $sth->fetchrow(); - $sth->finish(); - return $marcxml; + my ( $marc, $encoding ) = $dbh->selectrow_array(q| + SELECT marcxml, encoding + FROM import_records + WHERE import_record_id = ? + |, undef, $import_record_id ); + return $marc, $encoding; } =head2 AddImportBatch diff --git a/authorities/authorities.pl b/authorities/authorities.pl index fc4ae1a200..15a00eccb8 100755 --- a/authorities/authorities.pl +++ b/authorities/authorities.pl @@ -24,7 +24,7 @@ use CGI qw ( -utf8 ); use C4::Auth qw( get_template_and_user ); use C4::Output qw( output_html_with_http_headers ); use C4::AuthoritiesMarc qw( AddAuthority ModAuthority GetAuthority GetTagsLabels GetAuthMARCFromKohaField FindDuplicateAuthority ); -use C4::ImportBatch qw( GetImportRecordMarc ); +use C4::ImportBatch qw( GetImportRecordMarcXML ); use C4::Context; use Date::Calc qw( Today ); use MARC::File::USMARC; @@ -51,9 +51,11 @@ builds list, depending on authorised value... sub MARCfindbreeding_auth { my ( $id ) = @_; - my ($marc, $encoding) = GetImportRecordMarc($id); - if ($marc) { - my $record = MARC::Record->new_from_usmarc($marc); + my ($xml, $encoding) = GetImportRecordMarcXML($id); + if ($xml) { + my $marcflavour = C4::Context->preference('marcflavour'); + my $format = $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'; + my $record = MARC::Record->new_from_xml($xml, $encoding, $format); if ( !defined(ref($record)) ) { return -1; } else { diff --git a/catalogue/showmarc.pl b/catalogue/showmarc.pl index 8292bfe746..ab6c29487f 100755 --- a/catalogue/showmarc.pl +++ b/catalogue/showmarc.pl @@ -61,7 +61,9 @@ if(!ref $record) { } if($view eq 'card' || $view eq 'html') { - my $xml = $importid ? $record->as_xml(): GetXmlBiblio($biblionumber); + my $marcflavour = C4::Context->preference('marcflavour'); + my $format = $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'; + my $xml = $importid ? $record->as_xml($format): GetXmlBiblio($biblionumber); my $xsl; if ( $view eq 'card' ){ $xsl = C4::Context->preference('marcflavour') eq 'UNIMARC' -- 2.20.1