From 445f542df64eba18153d6182677538f441957af5 Mon Sep 17 00:00:00 2001 From: Jan Kissig Date: Tue, 10 Mar 2026 13:11:46 +0000 Subject: [PATCH] Bug 41681: (follow-up) Fix record number count when broken records are processed When running the script bulkmarcimport.pl, the number of MARC records reported at the end of the execution does not match the actual number of records processed from the input file. This patch fixes 3 issues - broken (invalid) records from batch get now counted (record_number) - when using the number param (n) to limit the number of records to be processed valid and invalid records get counted - when using th offset param (o) the scipt errored when broken records were part of the offset. To test: 1. Download the MARC file with the broken record 2. Run the following import commands and compare the counted import numbers: a) all records perl misc/migration_tools/bulkmarcimport.pl -b -v -m MARCXML --file 2_records_1st_broken.xml 4 MARC records done in 0.0421221256256104 seconds b) limit to 1 record in total perl misc/migration_tools/bulkmarcimport.pl -b -v -n 1 -m MARCXML --file 2_records_1st_broken.xml 4 MARC records done in 0.0420489311218262 seconds c) limit to 1 record offset 1 record perl misc/migration_tools/bulkmarcimport.pl -b -v -n 1 -o 1 -m MARCXML --file 2_records_1st_broken.xml :2: parser error : Opening and ending tag mismatch: leader2 line 2 and leader 00216nam a22001097a 4500 3. Apply the patch 4. Repeat steps in 2 and compare the results a) 2 MARC records done in 0.0405838489532471 seconds b) 1 MARC records done in 0.00948596000671387 seconds * c) 1 MARC records done in 0.0294840335845947 seconds *though this one was skipped --- misc/migration_tools/bulkmarcimport.pl | 46 ++++++++++++++++---------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index 5fef7e389da..8c7b00a645c 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -46,7 +46,8 @@ use Koha::SearchEngine::Search; use open qw( :std :encoding(UTF-8) ); binmode( STDOUT, ":encoding(UTF-8)" ); -my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 ); +my ( $input_marc_file, $offset, $cleanisbn ) = ( '', 0, 1 ); +my $number; my $version; my $delete; my $test_parameter; @@ -298,7 +299,7 @@ my $yamlhash; # Skip file offset if ($offset) { print "Skipping file offset: $offset records\n"; - $batch->next() while ( $offset-- ); + eval { $batch->next() } while ( $offset-- ); } my ( $tagid, $subfieldid ); @@ -333,22 +334,31 @@ RECORD: while () { my $record; - # get record - eval { $record = $batch->next() }; - if ($@) { - print "Bad MARC record $record_number: $@ skipped\n"; - - # FIXME - because MARC::Batch->next() combines grabbing the next - # blob and parsing it into one operation, a correctable condition - # such as a MARC-8 record claiming that it's UTF-8 can't be recovered - # from because we don't have access to the original blob. Note - # that the staging import can deal with this condition (via - # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. - next; + # max number (n) is defined and reached + $records_exhausted = 1 if defined($number) && $record_number >= $number; + + if ( !$records_exhausted ) { + + # get record + eval { $record = $batch->next() }; + if ($@) { + + # do increase number for broken records + $record_number++; + print "Bad MARC record $record_number: $@ skipped\n"; + + # FIXME - because MARC::Batch->next() combines grabbing the next + # blob and parsing it into one operation, a correctable condition + # such as a MARC-8 record claiming that it's UTF-8 can't be recovered + # from because we don't have access to the original blob. Note + # that the staging import can deal with this condition (via + # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. + next; + } } - if ($record) { - $record_number++; + if ( $record && !$records_exhausted ) { + $record_number++; if ($skip_bad_records) { my $xml = $record->as_xml_record(); eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); }; @@ -721,7 +731,7 @@ RECORD: while () { $records_exhausted = 1; } - if ( !$test_parameter && $record_number % $commitnum == 0 || $record_number == $number || $records_exhausted ) { + if ( !$test_parameter && $record_number % $commitnum == 0 || $records_exhausted ) { if ($indexer) { $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing; if ( C4::Context->preference('AutoLinkBiblios') ) { @@ -735,7 +745,7 @@ RECORD: while () { $schema->txn_commit; $schema->txn_begin; } - last if $record_number == $number || $records_exhausted; + last if $records_exhausted; } if ( !$test_parameter ) { -- 2.39.5