From 550e014cfe412127ebd2d6cf93872bdf6a097ace Mon Sep 17 00:00:00 2001 From: David Schmidt Date: Thu, 24 Mar 2022 12:30:56 +0100 Subject: [PATCH] Bug 30349: Tidy bulkmarcimport.pl to fix indentations Signed-off-by: Martin Renvoize Signed-off-by: Jonathan Druart --- misc/migration_tools/bulkmarcimport.pl | 564 +++++++++++++------------ 1 file changed, 303 insertions(+), 261 deletions(-) diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index 082fcfeff1d..c6039cb30e9 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -2,6 +2,7 @@ # Import an iso2709 file into Koha 3 use Modern::Perl; + #use diagnostics; # Koha modules used @@ -21,8 +22,8 @@ use C4::Biblio qw( BiblioAutoLink ); use C4::Koha; -use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); -use C4::Items qw( AddItemBatchFromMarc ); +use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); +use C4::Items qw( AddItemBatchFromMarc ); use C4::MarcModificationTemplates qw( GetModificationTemplates ModifyRecordWithTemplate @@ -30,11 +31,11 @@ use C4::MarcModificationTemplates qw( use C4::AuthoritiesMarc qw( GuessAuthTypeCode GuessAuthId GetAuthority ModAuthority AddAuthority ); use YAML::XS; -use Time::HiRes qw( gettimeofday ); +use Time::HiRes qw( gettimeofday ); use Getopt::Long qw( GetOptions ); use IO::File; use Pod::Usage qw( pod2usage ); -use FindBin (); +use FindBin (); use Koha::Logger; use Koha::Biblios; @@ -42,8 +43,8 @@ use Koha::SearchEngine; use Koha::SearchEngine::Search; use open qw( :std :encoding(UTF-8) ); -binmode(STDOUT, ":encoding(UTF-8)"); -my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1); +binmode( STDOUT, ":encoding(UTF-8)" ); +my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 ); my $version; my $delete; my $test_parameter; @@ -72,43 +73,43 @@ my $idmapfl; my $dedup_barcode; my $framework = ''; my $localcust; -my $marc_mod_template = ''; +my $marc_mod_template = ''; my $marc_mod_template_id = -1; $| = 1; GetOptions( - 'commit:f' => \$commit, - 'file:s' => \$input_marc_file, - 'n:f' => \$number, - 'o|offset:f' => \$offset, - 'h' => \$version, - 'd' => \$delete, - 't|test' => \$test_parameter, - 's' => \$skip_marc8_conversion, - 'c:s' => \$char_encoding, - 'v:+' => \$verbose, - 'fk' => \$fk_off, - 'm:s' => \$format, - 'l:s' => \$logfile, - 'append' => \$append, - 'k|keepids:s' => \$keepids, - 'b|biblios' => \$biblios, - 'a|authorities' => \$authorities, - 'authtypes:s' => \$authtypes, - 'filter=s@' => \$filters, - 'insert' => \$insert, - 'update' => \$update, - 'all' => \$all, - 'match=s@' => \$match, - 'i|isbn' => \$isbn_check, - 'x:s' => \$sourcetag, - 'y:s' => \$sourcesubfield, - 'idmap:s' => \$idmapfl, - 'cleanisbn!' => \$cleanisbn, - 'yaml:s' => \$yamlfile, - 'dedupbarcode' => \$dedup_barcode, - 'framework=s' => \$framework, - 'custom:s' => \$localcust, + 'commit:f' => \$commit, + 'file:s' => \$input_marc_file, + 'n:f' => \$number, + 'o|offset:f' => \$offset, + 'h' => \$version, + 'd' => \$delete, + 't|test' => \$test_parameter, + 's' => \$skip_marc8_conversion, + 'c:s' => \$char_encoding, + 'v:+' => \$verbose, + 'fk' => \$fk_off, + 'm:s' => \$format, + 'l:s' => \$logfile, + 'append' => \$append, + 'k|keepids:s' => \$keepids, + 'b|biblios' => \$biblios, + 'a|authorities' => \$authorities, + 'authtypes:s' => \$authtypes, + 'filter=s@' => \$filters, + 'insert' => \$insert, + 'update' => \$update, + 'all' => \$all, + 'match=s@' => \$match, + 'i|isbn' => \$isbn_check, + 'x:s' => \$sourcetag, + 'y:s' => \$sourcesubfield, + 'idmap:s' => \$idmapfl, + 'cleanisbn!' => \$cleanisbn, + 'yaml:s' => \$yamlfile, + 'dedupbarcode' => \$dedup_barcode, + 'framework=s' => \$framework, + 'custom:s' => \$localcust, 'marcmodtemplate:s' => \$marc_mod_template, ); @@ -116,21 +117,22 @@ $biblios ||= !$authorities; $insert ||= !$update; my $writemode = ($append) ? "a" : "w"; -pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities; +pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) + if $biblios && $authorities; if ($all) { $insert = 1; $update = 1; } -my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch'); -my $mod_biblio_options = { - disable_autolink => $using_elastic_search, +my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ); +my $mod_biblio_options = { + disable_autolink => $using_elastic_search, skip_record_index => $using_elastic_search, - overlay_context => { source => 'bulkmarcimport' } + overlay_context => { source => 'bulkmarcimport' } }; my $add_biblio_options = { - disable_autolink => $using_elastic_search, + disable_autolink => $using_elastic_search, skip_record_index => $using_elastic_search }; @@ -140,70 +142,70 @@ my $indexer; if ($using_elastic_search) { use Koha::SearchEngine::Elasticsearch::Indexer; $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( - { index => $authorities ? - $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX : - $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX + { + index => $authorities + ? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX + : $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); } -if ($version || ($input_marc_file eq '')) { +if ( $version || ( $input_marc_file eq '' ) ) { pod2usage( -verbose => 2 ); exit; } -if( $update && !( $match || $isbn_check ) ) { +if ( $update && !( $match || $isbn_check ) ) { warn "Using -update without -match or -isbn seems to be useless.\n"; } -if (defined $localcust) { #local customize module - if (!-e $localcust) { - $localcust = $localcust || 'LocalChanges'; #default name - $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only - $localcust =~ s/\.pm$//; #remove extension - my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir - if (-e $fqcust) { - $localcust= $fqcust; - } - else { +if ( defined $localcust ) { #local customize module + if ( !-e $localcust ) { + $localcust = $localcust || 'LocalChanges'; #default name + $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only + $localcust =~ s/\.pm$//; #remove extension + my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir + if ( -e $fqcust ) { + $localcust = $fqcust; + } else { print "WARNING: customize module $localcust.pm not found!\n"; exit 1; } } - require $localcust if $localcust; + require $localcust if $localcust; $localcust = \&customize if $localcust; } -if($marc_mod_template ne '') { +if ( $marc_mod_template ne '' ) { my @templates = GetModificationTemplates(); foreach my $this_template (@templates) { - if($this_template->{'name'} eq $marc_mod_template) { - if($marc_mod_template_id < 0) { + if ( $this_template->{'name'} eq $marc_mod_template ) { + if ( $marc_mod_template_id < 0 ) { $marc_mod_template_id = $this_template->{'template_id'}; } else { - print "WARNING: MARC modification template name " . - "'$marc_mod_template' matches multiple templates. " . - "Please rename these templates\n"; + print "WARNING: MARC modification template name " + . "'$marc_mod_template' matches multiple templates. " + . "Please rename these templates\n"; exit 1; } } } - if($marc_mod_template_id < 0) { + if ( $marc_mod_template_id < 0 ) { die "Can't located MARC modification template '$marc_mod_template'\n"; } else { print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose; } } -my $dbh = C4::Context->dbh; +my $dbh = C4::Context->dbh; my $heading_fields = get_heading_fields(); my $idmapfh; -if (defined $idmapfl) { - open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; +if ( defined $idmapfl ) { + open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n"; } -if ((not defined $sourcesubfield) && (not defined $sourcetag)) { - $sourcetag = "910"; +if ( ( not defined $sourcesubfield ) && ( not defined $sourcetag ) ) { + $sourcetag = "910"; $sourcesubfield = "a"; } @@ -225,8 +227,7 @@ if ($delete) { $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); $dbh->do("DELETE FROM items"); $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); - } - else { + } else { print "Deleting authorities\n"; $dbh->do("truncate auth_header"); } @@ -244,7 +245,7 @@ my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; my $searcher = Koha::SearchEngine::Search->new( { index => ( - $authorities + $authorities ? $Koha::SearchEngine::AUTHORITIES_INDEX : $Koha::SearchEngine::BIBLIOS_INDEX ) @@ -254,8 +255,9 @@ my $searcher = Koha::SearchEngine::Search->new( print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; my $starttime = gettimeofday; -my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer -if (defined $format && $format =~ /XML/i) { +my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer +if ( defined $format && $format =~ /XML/i ) { + # ugly hack follows -- MARC::File::XML, when used by MARC::Batch, # appears to try to convert incoming XML records from MARC-8 # to UTF-8. Setting the BinaryEncoding key turns that off @@ -266,14 +268,14 @@ if (defined $format && $format =~ /XML/i) { # extract the records, not using regexes to look # for .*. $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; - my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour)); + my $recordformat = ( $marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour) ); + #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. - $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21"); + $recordformat = $recordformat . "AUTH" if ( $authorities and $marc_flavour ne "MARC21" ); $MARC::File::XML::_load_args{RecordFormat} = $recordformat; - $batch = MARC::Batch->new('XML', $fh); -} -else { - $batch = MARC::Batch->new('USMARC', $fh); + $batch = MARC::Batch->new( 'XML', $fh ); +} else { + $batch = MARC::Batch->new( 'USMARC', $fh ); } $batch->warnings_off(); @@ -284,19 +286,19 @@ my $yamlhash; # Skip file offset if ($offset) { print "Skipping file offset: $offset records\n"; - $batch->next() while ($offset--); + $batch->next() while ( $offset-- ); } -my ($tagid, $subfieldid); +my ( $tagid, $subfieldid ); if ($authorities) { $tagid = '001'; -} -else { - ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber"); +} else { + ( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber"); $tagid ||= "001"; } my $sth_isbn; + # the SQL query to search on isbn if ($isbn_check) { $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); @@ -304,21 +306,23 @@ if ($isbn_check) { my $loghandle; if ($logfile) { - $loghandle= IO::File->new($logfile, $writemode); + $loghandle = IO::File->new( $logfile, $writemode ); print $loghandle "id;operation;status\n"; } my $record_number = 0; -my $logger = Koha::Logger->get; -my $schema = Koha::Database->schema; -my $marc_records = []; -RECORD: while ( ) { +my $logger = Koha::Logger->get; +my $schema = Koha::Database->schema; +my $marc_records = []; +RECORD: while () { my $record; $record_number++; + # get record eval { $record = $batch->next() }; if ($@) { print "Bad MARC record $record_number: $@ skipped\n"; + # FIXME - because MARC::Batch->next() combines grabbing the next # blob and parsing it into one operation, a correctable condition # such as a MARC-8 record claiming that it's UTF-8 can't be recovered @@ -328,11 +332,15 @@ RECORD: while ( ) { next; } if ($record) { + # transcode the record to UTF8 if needed & applicable. - if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { - my ($guessed_charset, $charset_errors); - ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : '')); - if ($guessed_charset eq 'failed') { + if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) { + my ( $guessed_charset, $charset_errors ); + ( $record, $guessed_charset, $charset_errors ) = MarcToUTF8Record( + $record, + $marc_flavour . ( ( $authorities and $marc_flavour ne "MARC21" ) ? 'AUTH' : '' ) + ); + if ( $guessed_charset eq 'failed' ) { warn "ERROR: failed to perform character conversion for record $record_number\n"; next RECORD; } @@ -340,8 +348,7 @@ RECORD: while ( ) { SetUTF8Flag($record); &$localcust($record) if $localcust; push @{$marc_records}, $record; - } - else { + } else { last; } } @@ -349,55 +356,60 @@ RECORD: while ( ) { $record_number = 0; my $records_total = @{$marc_records}; $schema->txn_begin; -RECORD: foreach my $record (@{$marc_records}) { +RECORD: foreach my $record ( @{$marc_records} ) { $record_number++; - if (($verbose//1) == 1) { #no dot for verbose==2 - print "." . ($record_number % 100 == 0 ? "\n$record_number" : ''); + if ( ( $verbose // 1 ) == 1 ) { #no dot for verbose==2 + print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' ); } - if ($marc_mod_template_id > 0) { + if ( $marc_mod_template_id > 0 ) { print "Modifying MARC\n" if $verbose; ModifyRecordWithTemplate( $marc_mod_template_id, $record ); } my $isbn; + # remove trailing - in isbn (only for biblios, of course) - if ($biblios && ($cleanisbn || $isbn_check)) { - my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; + if ( $biblios && ( $cleanisbn || $isbn_check ) ) { + my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; my $field = $record->field($tag); $isbn = $field && $field->subfield('a'); - if ($isbn && $cleanisbn) { + if ( $isbn && $cleanisbn ) { $isbn =~ s/-//g; - $field->update('a' => $isbn); + $field->update( 'a' => $isbn ); } } + # search for duplicates (based on Local-number) - my $originalid = GetRecordId($record, $tagid, $subfieldid); + my $originalid = GetRecordId( $record, $tagid, $subfieldid ); my $matched_record_id = undef; if ($match) { require C4::Search; - my $server = ($authorities ? 'authorityserver' : 'biblioserver'); - my $query = build_query($match, $record); + my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); + my $query = build_query( $match, $record ); $logger->debug("Bulkmarcimport: $query"); - my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]); + my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); + # changed to warn so able to continue with one broken record - if (defined $error) { + if ( defined $error ) { warn "unable to search the database for duplicates : $error"; - printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile); + printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile); next RECORD; } $logger->debug("Bulkmarcimport: $query $server : $totalhits"); + # sub SimpleSearch could return undefined, but only on error, so # should not really need to safeguard here, but do so anyway $results //= []; - if (@{$results} == 1) { - my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]); + if ( @{$results} == 1 ) { + my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] ); SetUTF8Flag($matched_record); - $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid); + $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid ); + + if ( $authorities && $marc_flavour ) { - if ($authorities && $marc_flavour) { #Skip if authority in database is the same or newer than the incoming record - if (RecordRevisionIsGtOrEq($matched_record, $record)) { + if ( RecordRevisionIsGtOrEq( $matched_record, $record ) ) { if ($yamlfile) { $yamlhash->{$originalid} = YAMLFileEntry( $matched_record, @@ -408,257 +420,279 @@ RECORD: foreach my $record (@{$marc_records}) { next; } } - } - elsif(@{$results} > 1) { + } elsif ( @{$results} > 1 ) { $logger->debug("More than one match for: $query"); - } - else { + } else { $logger->debug("No match for: $query"); } - if ($keepids && $originalid) { + if ( $keepids && $originalid ) { my $storeidfield; - if (length($keepids) == 3) { - $storeidfield = MARC::Field->new($keepids, $originalid); + if ( length($keepids) == 3 ) { + $storeidfield = MARC::Field->new( $keepids, $originalid ); } else { - $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid); + $storeidfield = + MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); } $record->insert_fields_ordered($storeidfield); - $record->delete_field($record->field($tagid)); + $record->delete_field( $record->field($tagid) ); } } foreach my $stringfilter (@$filters) { - if (length($stringfilter) == 3) { - foreach my $field ($record->field($stringfilter)) { + if ( length($stringfilter) == 3 ) { + foreach my $field ( $record->field($stringfilter) ) { $record->delete_field($field); - $logger->debug("Removed: ", $field->as_string); + $logger->debug( "Removed: ", $field->as_string ); } - } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { - my $removetag = $1; + } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) { + my $removetag = $1; my $removesubfield = $2; - my $removematch = $3; - if (($removetag > "010") && $removesubfield) { - foreach my $field ($record->field($removetag)) { - $field->delete_subfield(code => "$removesubfield", match => $removematch); - $logger->debug("Potentially removed: ", $field->subfield($removesubfield)); + my $removematch = $3; + if ( ( $removetag > "010" ) && $removesubfield ) { + foreach my $field ( $record->field($removetag) ) { + $field->delete_subfield( code => "$removesubfield", match => $removematch ); + $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) ); } } } } unless ($test_parameter) { if ($authorities) { - my $authtypecode = GuessAuthTypeCode($record, $heading_fields); + my $authtypecode = GuessAuthTypeCode( $record, $heading_fields ); my $authid; if ($matched_record_id) { if ($update) { ## Authority has an id and is in database: update - eval { ($authid) = ModAuthority($matched_record_id, $record, $authtypecode) }; + eval { ($authid) = ModAuthority( $matched_record_id, $record, $authtypecode ) }; if ($@) { warn "ERROR: Update authority $matched_record_id failed: $@\n"; - printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if ($logfile); - } - else { - printlog({ id => $authid, op => "update", status => "ok" }) if ($logfile); + printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); + } else { + printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile); } - } - elsif ($logfile) { + } elsif ($logfile) { warn "WARNING: Update authority $originalid skipped"; - printlog({ - id => $matched_record_id, - op => "update", - status => "warning: authority already in database and option -update not enabled, skipping..." - }); + printlog( + { + id => $matched_record_id, + op => "update", + status => + "warning: authority already in database and option -update not enabled, skipping..." + } + ); } - } - elsif ($insert) { + } elsif ($insert) { ## An authid is defined but no authority in database: insert - eval { ($authid) = AddAuthority($record, undef, $authtypecode) }; + eval { ($authid) = AddAuthority( $record, undef, $authtypecode ) }; if ($@) { warn "ERROR: Insert authority $originalid failed: $@\n"; - printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile); - } - else { - printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile); + printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); + } else { + printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile); } - } - else { + } else { warn "WARNING: Insert authority $originalid skipped"; - printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile); + printlog( + { + id => $originalid, op => "insert", + status => "warning : biblio not in database and option -insert not enabled, skipping..." + } + ) if ($logfile); } if ($yamlfile) { $yamlhash->{$originalid} = YAMLFileEntry( $record, $authid, - 1 #@FIXME: Really always updated? + 1 #@FIXME: Really always updated? ); } - } - else { - my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id); + } else { + my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id ); + # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter - if (!$matched_record_id && $isbn_check && $isbn) { + if ( !$matched_record_id && $isbn_check && $isbn ) { $sth_isbn->execute($isbn); - ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow; + ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow; } - if (defined $idmapfl && $matched_record_id) { - if ($sourcetag < "010") { - if ($record->field($sourcetag)) { + if ( defined $idmapfl && $matched_record_id ) { + if ( $sourcetag < "010" ) { + if ( $record->field($sourcetag) ) { my $source = $record->field($sourcetag)->data(); - printf($idmapfh "%s|%s\n", $source, $matched_record_id); + printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); } - } - else { - my $source = $record->subfield($sourcetag, $sourcesubfield); - printf($idmapfh "%s|%s\n", $source, $matched_record_id); + } else { + my $source = $record->subfield( $sourcetag, $sourcesubfield ); + printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); } } # Create biblio, unless we already have it (either match or ISBN) if ($matched_record_id) { - eval{ - $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber; - }; + eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; }; if ($update) { my $success; - eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $mod_biblio_options) }; + eval { + $success = ModBiblio( + $record, $matched_record_id, GetFrameworkCode($matched_record_id), + $mod_biblio_options + ); + }; if ($@) { warn "ERROR: Update biblio $matched_record_id failed: $@\n"; printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); next RECORD; - } - elsif (!$success) { + } elsif ( !$success ) { warn "ERROR: Update biblio $matched_record_id failed for unknown reason"; printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); next RECORD; - } - else { + } else { $record_id = $matched_record_id; printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile); } - } - else { + } else { warn "WARNING: Update biblio $originalid skipped"; - printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile); + printlog( + { + id => $matched_record_id, op => "update", + status => "warning : already in database and option -update not enabled, skipping..." + } + ) if ($logfile); } - } - elsif ($insert) { + } elsif ($insert) { my $record_clone = $record->clone(); C4::Biblio::_strip_item_fields($record_clone); - eval { ($record_id, $biblioitemnumber) = AddBiblio($record_clone, $framework, $add_biblio_options) }; + eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) }; if ($@) { warn "ERROR: Insert biblio $originalid failed: $@\n"; printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); next RECORD; - } - else { + } else { printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile); } + # If incoming record has bib ids set we need to transfer # new ids from record_clone to incoming record to avoid # working on wrong record (the original record) later on # when adding items for example - C4::Biblio::_koha_marc_update_bib_ids($record, $framework, $record_id, $biblioitemnumber); - } - else { + C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber ); + } else { warn "WARNING: Insert biblio $originalid skipped"; - printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile); + printlog( + { + id => $originalid, op => "insert", + status => "warning : biblio not in database and option -insert not enabled, skipping..." + } + ) if ($logfile); next RECORD; } my $record_has_added_items = 0; if ($record_id) { $yamlhash->{$originalid} = $record_id if $yamlfile; - eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); }; + eval { + ( $itemnumbers_ref, $errors_ref ) = + AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework ); + }; my $error_adding = $@; $record_has_added_items = @{$itemnumbers_ref}; if ($error_adding) { warn "ERROR: Adding items to bib $record_id failed: $error_adding"; - printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile); + printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); + # if we failed because of an exception, assume that # the MARC columns in biblioitems were not set. next RECORD; } - if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { + if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } + @$errors_ref ) + { # Find the record called 'barcode' - my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode'); + my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode'); + # Now remove any items that didn't have a duplicate_barcode error, # erase the barcodes on items that did, and re-add those items. my %dupes; - foreach my $i (0 .. $#{$errors_ref}) { + foreach my $i ( 0 .. $#{$errors_ref} ) { my $ref = $errors_ref->[$i]; - if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { - $dupes{$ref->{item_sequence}} = 1; + if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) { + $dupes{ $ref->{item_sequence} } = 1; + # Delete the error message because we're going to # retry this one. delete $errors_ref->[$i]; } } my $seq = 0; - foreach my $field ($record->field($tag)) { + foreach my $field ( $record->field($tag) ) { $seq++; - if ($dupes{$seq}) { + if ( $dupes{$seq} ) { + # Here we remove the barcode - $field->delete_subfield(code => $sub); - } - else { + $field->delete_subfield( code => $sub ); + } else { + # otherwise we delete the field because we don't want # two of them $record->delete_fields($field); } } + # Now re-add the record as before, adding errors to the prev list my $more_errors; - eval { ($itemnumbers_ref, $more_errors) = - AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); }; + eval { + ( $itemnumbers_ref, $more_errors ) = + AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' ); + }; if ($@) { warn "ERROR: Adding items to bib $record_id failed: $@\n"; - printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile); + printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); + # if we failed because of an exception, assume that # the MARC columns in biblioitems were not set. next RECORD; } $record_has_added_items ||= @{$itemnumbers_ref}; - if (@{$more_errors}) { + if ( @{$more_errors} ) { push @$errors_ref, @{$more_errors}; } } - if ($record_has_added_items) { - printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile); + if ($record_has_added_items) { + printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile); } - if (@{$errors_ref}) { - report_item_errors($record_id, $errors_ref); + if ( @{$errors_ref} ) { + report_item_errors( $record_id, $errors_ref ); } my $biblio = Koha::Biblios->find($record_id); $record = $biblio->metadata->record( { embed_items => 1 } ); push @search_engine_record_ids, $record_id; - push @search_engine_records, $record; + push @search_engine_records, $record; } } - if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) { + if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) { $schema->txn_commit; $schema->txn_begin; if ($indexer) { - $indexer->update_index(\@search_engine_record_ids, \@search_engine_records); - if (C4::Context->preference('AutoLinkBiblios')) { + $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ); + if ( C4::Context->preference('AutoLinkBiblios') ) { foreach my $record (@search_engine_records) { - BiblioAutoLink($record, $framework); + BiblioAutoLink( $record, $framework ); } } @search_engine_record_ids = (); - @search_engine_records = (); + @search_engine_records = (); } } } - print $record->as_formatted() . "\n" if ($verbose//0) == 2; - last if $record_number == $number; + print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2; + last if $record_number == $number; } $schema->txn_commit; @@ -679,104 +713,112 @@ if ($logfile) { } if ($yamlfile) { open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; - print $yamlfileout Encode::decode_utf8(YAML::XS::Dump($yamlhash)); + print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) ); } exit 0; sub YAMLFileEntry { - my ($record, $record_id, $updated) = @_; + my ( $record, $record_id, $updated ) = @_; - my $entry = { - authid => $record_id - }; + my $entry = { authid => $record_id }; # we recover all subfields of the heading authorities my @subfields; - foreach my $field ($record->field("2..")) { + foreach my $field ( $record->field("2..") ) { push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); } $entry->{'subfields'} = \@subfields; - $entry->{'updated'} = $updated; + $entry->{'updated'} = $updated; return $entry; } sub RecordRevisionIsGtOrEq { - my ($record_a, $record_b) = @_; - return $record_a->field('005') && $record_b->field('005') && - $record_a->field('005')->data && $record_b->field('005')->data && - $record_a->field('005')->data >= $record_b->field('005')->data; + my ( $record_a, $record_b ) = @_; + return + $record_a->field('005') + && $record_b->field('005') + && $record_a->field('005')->data + && $record_b->field('005')->data + && $record_a->field('005')->data >= $record_b->field('005')->data; } sub GetRecordId { my $marcrecord = shift; - my $tag = shift; - my $subfield = shift; - if ($tag lt "010") { + my $tag = shift; + my $subfield = shift; + if ( $tag lt "010" ) { return $marcrecord->field($tag)->data() if $marcrecord->field($tag); - } - elsif ($subfield) { - if ($marcrecord->field($tag)) { - return $marcrecord->subfield($tag, $subfield); + } elsif ($subfield) { + if ( $marcrecord->field($tag) ) { + return $marcrecord->subfield( $tag, $subfield ); } } } + sub build_query { - my ($match, $record) = @_; + my ( $match, $record ) = @_; my @searchstrings; foreach my $matchpoint (@$match) { - my $query = build_simplequery($matchpoint, $record); - push (@searchstrings, $query) if $query; + my $query = build_simplequery( $matchpoint, $record ); + push( @searchstrings, $query ) if $query; } my $op = 'AND'; - return join(" $op ", @searchstrings); + return join( " $op ", @searchstrings ); } + sub build_simplequery { - my ($matchpoint, $record) = @_; + my ( $matchpoint, $record ) = @_; my @searchstrings; - my ($index, $record_data) = split (/,/, $matchpoint); - if ($record_data =~ /(\d{3})(.*)/) { - my ($tag, $subfields) = ($1, $2); - foreach my $field ($record->field($tag)) { - if (length($field->as_string("$subfields")) > 0) { - push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\""); + my ( $index, $record_data ) = split( /,/, $matchpoint ); + if ( $record_data =~ /(\d{3})(.*)/ ) { + my ( $tag, $subfields ) = ( $1, $2 ); + foreach my $field ( $record->field($tag) ) { + if ( length( $field->as_string("$subfields") ) > 0 ) { + push( @searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"" ); } } - } - else { + } else { print "Invalid matchpoint format, invalid marc-field: $matchpoint\n"; } my $op = 'AND'; - return join(" $op ", @searchstrings); + return join( " $op ", @searchstrings ); } + sub report_item_errors { my $biblionumber = shift; - my $errors_ref = shift; + my $errors_ref = shift; - foreach my $error (@{ $errors_ref }) { + foreach my $error ( @{$errors_ref} ) { next if !$error; - my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): "; + my $msg = + "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): "; my $error_code = $error->{'error_code'}; $error_code =~ s/_/ /g; $msg .= "$error_code $error->{'error_information'}"; print $msg, "\n"; } } + sub printlog { my $logelements = shift; - print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw}), "\n"; + print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw} ), "\n"; } + sub get_heading_fields { my $headingfields; if ($authtypes) { $headingfields = YAML::XS::LoadFile($authtypes); $headingfields = { C4::Context->preference('marcflavour') => $headingfields }; - $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); + $logger->debug( Encode::decode_utf8( YAML::XS::Dump($headingfields) ) ); } unless ($headingfields) { - $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); + $headingfields = $dbh->selectall_hashref( + "SELECT auth_tag_to_report, authtypecode from auth_types", 'auth_tag_to_report', + { Slice => {} } + ); $headingfields = { C4::Context->preference('marcflavour') => $headingfields }; } return $headingfields; -- 2.34.1