From 96c0261a0b5f6efcf99fe82e87f94bfe3a4bd3be Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 25 Jul 2024 16:37:17 +0000 Subject: [PATCH] Bug 37478: Add strict mode to bulkmarcimport This patch adds a conversion from MARC -> XML -> MARC to catch any parsing errors. If errors are found, we then lint the record to catch any problems, output the warnings, and skip the record. To test: 1 - Download the sample records from this bug report 2 - perl misc/migration_tools/bulkmarcimport.pl -b --file=520_nosubfield.mrc -v fails! 3 - perl misc/migration_tools/bulkmarcimport.pl -b --file=003_subfielda.mrc -v fails! 4 - Apply patch 5 - Repeat 2 & 3 - no change 6 - Add -st switch to the commands: perl misc/migration_tools/bulkmarcimport.pl -b --file=520_nosubfield.mrc -v -st perl misc/migration_tools/bulkmarcimport.pl -b --file=003_subfielda.mrc -v -st 7 - The records are now skipped, and the script completes 8 - Confirm the warnings generated are useful 9 - Sign off! --- misc/migration_tools/bulkmarcimport.pl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index 8161a0bb0ad..eaed2a2890d 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -9,6 +9,7 @@ use Modern::Perl; use MARC::File::USMARC; use MARC::File::XML; use MARC::Batch; +use MARC::Lint; use Encode; use Koha::Script; @@ -75,6 +76,7 @@ my $framework = ''; my $localcust; my $marc_mod_template = ''; my $marc_mod_template_id = -1; +my $strict_mode; $| = 1; GetOptions( @@ -111,6 +113,7 @@ GetOptions( 'framework=s' => \$framework, 'custom:s' => \$localcust, 'marcmodtemplate:s' => \$marc_mod_template, + 'st|strict' => \$strict_mode, ); $biblios ||= !$authorities; @@ -314,6 +317,7 @@ my $record_number = 0; my $logger = Koha::Logger->get; my $schema = Koha::Database->schema; my $marc_records = []; +my $lint = MARC::Lint->new; RECORD: while () { my $record; $record_number++; @@ -333,6 +337,19 @@ RECORD: while () { } if ($record) { + if ($strict_mode) { + my $xml = $record->as_xml_record(); + eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); }; + if ($@) { + print "Record $record_number generated invalid xml:\n"; + $lint->check_record($record); + foreach my $warning ( $lint->warnings ) { + print " " . $warning . "\n"; + } + print " Record skipped!"; + next; + } + } # transcode the record to UTF8 if needed & applicable. if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) { my ( $guessed_charset, $charset_errors ); -- 2.39.2