Bugzilla – Attachment 160492 Details for
Bug 30349
Cleanup bulkmarcimport.pl
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 30349: Tidy bulkmarcimport.pl to fix indentations
Bug-30349-Tidy-bulkmarcimportpl-to-fix-indentation.patch (text/plain), 42.58 KB, created by
Martin Renvoize (ashimema)
on 2024-01-03 16:56:20 UTC
(
hide
)
Description:
Bug 30349: Tidy bulkmarcimport.pl to fix indentations
Filename:
MIME Type:
Creator:
Martin Renvoize (ashimema)
Created:
2024-01-03 16:56:20 UTC
Size:
42.58 KB
patch
obsolete
>From 33d37f4b8251c28ea172149a4aa7399e7a9df5a3 Mon Sep 17 00:00:00 2001 >From: David Schmidt <mail@davidschmidt.at> >Date: Thu, 24 Mar 2022 12:30:56 +0100 >Subject: [PATCH] Bug 30349: Tidy bulkmarcimport.pl to fix indentations > >Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com> >--- > misc/migration_tools/bulkmarcimport.pl | 564 +++++++++++++------------ > 1 file changed, 303 insertions(+), 261 deletions(-) > >diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl >index 082fcfeff1d..c6039cb30e9 100755 >--- a/misc/migration_tools/bulkmarcimport.pl >+++ b/misc/migration_tools/bulkmarcimport.pl >@@ -2,6 +2,7 @@ > # Import an iso2709 file into Koha 3 > > use Modern::Perl; >+ > #use diagnostics; > > # Koha modules used >@@ -21,8 +22,8 @@ use C4::Biblio qw( > BiblioAutoLink > ); > use C4::Koha; >-use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); >-use C4::Items qw( AddItemBatchFromMarc ); >+use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); >+use C4::Items qw( AddItemBatchFromMarc ); > use C4::MarcModificationTemplates qw( > GetModificationTemplates > ModifyRecordWithTemplate >@@ -30,11 +31,11 @@ use C4::MarcModificationTemplates qw( > use C4::AuthoritiesMarc qw( GuessAuthTypeCode GuessAuthId GetAuthority ModAuthority AddAuthority ); > > use YAML::XS; >-use Time::HiRes qw( gettimeofday ); >+use Time::HiRes qw( gettimeofday ); > use Getopt::Long qw( GetOptions ); > use IO::File; > use Pod::Usage qw( pod2usage ); >-use FindBin (); >+use FindBin (); > > use Koha::Logger; > use Koha::Biblios; >@@ -42,8 +43,8 @@ use Koha::SearchEngine; > use Koha::SearchEngine::Search; > > use open qw( :std :encoding(UTF-8) ); >-binmode(STDOUT, ":encoding(UTF-8)"); >-my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1); >+binmode( STDOUT, ":encoding(UTF-8)" ); >+my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 ); > my $version; > my $delete; > my $test_parameter; >@@ -72,43 +73,43 @@ my $idmapfl; > my $dedup_barcode; > my $framework = ''; > my $localcust; >-my $marc_mod_template = ''; >+my $marc_mod_template = ''; > my $marc_mod_template_id = -1; > $| = 1; > > GetOptions( >- 'commit:f' => \$commit, >- 'file:s' => \$input_marc_file, >- 'n:f' => \$number, >- 'o|offset:f' => \$offset, >- 'h' => \$version, >- 'd' => \$delete, >- 't|test' => \$test_parameter, >- 's' => \$skip_marc8_conversion, >- 'c:s' => \$char_encoding, >- 'v:+' => \$verbose, >- 'fk' => \$fk_off, >- 'm:s' => \$format, >- 'l:s' => \$logfile, >- 'append' => \$append, >- 'k|keepids:s' => \$keepids, >- 'b|biblios' => \$biblios, >- 'a|authorities' => \$authorities, >- 'authtypes:s' => \$authtypes, >- 'filter=s@' => \$filters, >- 'insert' => \$insert, >- 'update' => \$update, >- 'all' => \$all, >- 'match=s@' => \$match, >- 'i|isbn' => \$isbn_check, >- 'x:s' => \$sourcetag, >- 'y:s' => \$sourcesubfield, >- 'idmap:s' => \$idmapfl, >- 'cleanisbn!' => \$cleanisbn, >- 'yaml:s' => \$yamlfile, >- 'dedupbarcode' => \$dedup_barcode, >- 'framework=s' => \$framework, >- 'custom:s' => \$localcust, >+ 'commit:f' => \$commit, >+ 'file:s' => \$input_marc_file, >+ 'n:f' => \$number, >+ 'o|offset:f' => \$offset, >+ 'h' => \$version, >+ 'd' => \$delete, >+ 't|test' => \$test_parameter, >+ 's' => \$skip_marc8_conversion, >+ 'c:s' => \$char_encoding, >+ 'v:+' => \$verbose, >+ 'fk' => \$fk_off, >+ 'm:s' => \$format, >+ 'l:s' => \$logfile, >+ 'append' => \$append, >+ 'k|keepids:s' => \$keepids, >+ 'b|biblios' => \$biblios, >+ 'a|authorities' => \$authorities, >+ 'authtypes:s' => \$authtypes, >+ 'filter=s@' => \$filters, >+ 'insert' => \$insert, >+ 'update' => \$update, >+ 'all' => \$all, >+ 'match=s@' => \$match, >+ 'i|isbn' => \$isbn_check, >+ 'x:s' => \$sourcetag, >+ 'y:s' => \$sourcesubfield, >+ 'idmap:s' => \$idmapfl, >+ 'cleanisbn!' => \$cleanisbn, >+ 'yaml:s' => \$yamlfile, >+ 'dedupbarcode' => \$dedup_barcode, >+ 'framework=s' => \$framework, >+ 'custom:s' => \$localcust, > 'marcmodtemplate:s' => \$marc_mod_template, > ); > >@@ -116,21 +117,22 @@ $biblios ||= !$authorities; > $insert ||= !$update; > my $writemode = ($append) ? "a" : "w"; > >-pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities; >+pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) >+ if $biblios && $authorities; > > if ($all) { > $insert = 1; > $update = 1; > } > >-my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch'); >-my $mod_biblio_options = { >- disable_autolink => $using_elastic_search, >+my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ); >+my $mod_biblio_options = { >+ disable_autolink => $using_elastic_search, > skip_record_index => $using_elastic_search, >- overlay_context => { source => 'bulkmarcimport' } >+ overlay_context => { source => 'bulkmarcimport' } > }; > my $add_biblio_options = { >- disable_autolink => $using_elastic_search, >+ disable_autolink => $using_elastic_search, > skip_record_index => $using_elastic_search > }; > >@@ -140,70 +142,70 @@ my $indexer; > if ($using_elastic_search) { > use Koha::SearchEngine::Elasticsearch::Indexer; > $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( >- { index => $authorities ? >- $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX : >- $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX >+ { >+ index => $authorities >+ ? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX >+ : $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX > } > ); > } > >-if ($version || ($input_marc_file eq '')) { >+if ( $version || ( $input_marc_file eq '' ) ) { > pod2usage( -verbose => 2 ); > exit; > } >-if( $update && !( $match || $isbn_check ) ) { >+if ( $update && !( $match || $isbn_check ) ) { > warn "Using -update without -match or -isbn seems to be useless.\n"; > } > >-if (defined $localcust) { #local customize module >- if (!-e $localcust) { >- $localcust = $localcust || 'LocalChanges'; #default name >- $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only >- $localcust =~ s/\.pm$//; #remove extension >- my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir >- if (-e $fqcust) { >- $localcust= $fqcust; >- } >- else { >+if ( defined $localcust ) { #local customize module >+ if ( !-e $localcust ) { >+ $localcust = $localcust || 'LocalChanges'; #default name >+ $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only >+ $localcust =~ s/\.pm$//; #remove extension >+ my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir >+ if ( -e $fqcust ) { >+ $localcust = $fqcust; >+ } else { > print "WARNING: customize module $localcust.pm not found!\n"; > exit 1; > } > } >- require $localcust if $localcust; >+ require $localcust if $localcust; > $localcust = \&customize if $localcust; > } > >-if($marc_mod_template ne '') { >+if ( $marc_mod_template ne '' ) { > my @templates = GetModificationTemplates(); > foreach my $this_template (@templates) { >- if($this_template->{'name'} eq $marc_mod_template) { >- if($marc_mod_template_id < 0) { >+ if ( $this_template->{'name'} eq $marc_mod_template ) { >+ if ( $marc_mod_template_id < 0 ) { > $marc_mod_template_id = $this_template->{'template_id'}; > } else { >- print "WARNING: MARC modification template name " . >- "'$marc_mod_template' matches multiple templates. " . >- "Please rename these templates\n"; >+ print "WARNING: MARC modification template name " >+ . "'$marc_mod_template' matches multiple templates. " >+ . "Please rename these templates\n"; > exit 1; > } > } > } >- if($marc_mod_template_id < 0) { >+ if ( $marc_mod_template_id < 0 ) { > die "Can't located MARC modification template '$marc_mod_template'\n"; > } else { > print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose; > } > } > >-my $dbh = C4::Context->dbh; >+my $dbh = C4::Context->dbh; > my $heading_fields = get_heading_fields(); > my $idmapfh; > >-if (defined $idmapfl) { >- open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; >+if ( defined $idmapfl ) { >+ open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n"; > } > >-if ((not defined $sourcesubfield) && (not defined $sourcetag)) { >- $sourcetag = "910"; >+if ( ( not defined $sourcesubfield ) && ( not defined $sourcetag ) ) { >+ $sourcetag = "910"; > $sourcesubfield = "a"; > } > >@@ -225,8 +227,7 @@ if ($delete) { > $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); > $dbh->do("DELETE FROM items"); > $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); >- } >- else { >+ } else { > print "Deleting authorities\n"; > $dbh->do("truncate auth_header"); > } >@@ -244,7 +245,7 @@ my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; > my $searcher = Koha::SearchEngine::Search->new( > { > index => ( >- $authorities >+ $authorities > ? $Koha::SearchEngine::AUTHORITIES_INDEX > : $Koha::SearchEngine::BIBLIOS_INDEX > ) >@@ -254,8 +255,9 @@ my $searcher = Koha::SearchEngine::Search->new( > print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; > my $starttime = gettimeofday; > >-my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer >-if (defined $format && $format =~ /XML/i) { >+my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer >+if ( defined $format && $format =~ /XML/i ) { >+ > # ugly hack follows -- MARC::File::XML, when used by MARC::Batch, > # appears to try to convert incoming XML records from MARC-8 > # to UTF-8. Setting the BinaryEncoding key turns that off >@@ -266,14 +268,14 @@ if (defined $format && $format =~ /XML/i) { > # extract the records, not using regexes to look > # for <record>.*</record>. > $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; >- my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour)); >+ my $recordformat = ( $marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour) ); >+ > #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. >- $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21"); >+ $recordformat = $recordformat . "AUTH" if ( $authorities and $marc_flavour ne "MARC21" ); > $MARC::File::XML::_load_args{RecordFormat} = $recordformat; >- $batch = MARC::Batch->new('XML', $fh); >-} >-else { >- $batch = MARC::Batch->new('USMARC', $fh); >+ $batch = MARC::Batch->new( 'XML', $fh ); >+} else { >+ $batch = MARC::Batch->new( 'USMARC', $fh ); > } > > $batch->warnings_off(); >@@ -284,19 +286,19 @@ my $yamlhash; > # Skip file offset > if ($offset) { > print "Skipping file offset: $offset records\n"; >- $batch->next() while ($offset--); >+ $batch->next() while ( $offset-- ); > } > >-my ($tagid, $subfieldid); >+my ( $tagid, $subfieldid ); > if ($authorities) { > $tagid = '001'; >-} >-else { >- ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber"); >+} else { >+ ( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber"); > $tagid ||= "001"; > } > > my $sth_isbn; >+ > # the SQL query to search on isbn > if ($isbn_check) { > $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); >@@ -304,21 +306,23 @@ if ($isbn_check) { > > my $loghandle; > if ($logfile) { >- $loghandle= IO::File->new($logfile, $writemode); >+ $loghandle = IO::File->new( $logfile, $writemode ); > print $loghandle "id;operation;status\n"; > } > > my $record_number = 0; >-my $logger = Koha::Logger->get; >-my $schema = Koha::Database->schema; >-my $marc_records = []; >-RECORD: while ( ) { >+my $logger = Koha::Logger->get; >+my $schema = Koha::Database->schema; >+my $marc_records = []; >+RECORD: while () { > my $record; > $record_number++; >+ > # get record > eval { $record = $batch->next() }; > if ($@) { > print "Bad MARC record $record_number: $@ skipped\n"; >+ > # FIXME - because MARC::Batch->next() combines grabbing the next > # blob and parsing it into one operation, a correctable condition > # such as a MARC-8 record claiming that it's UTF-8 can't be recovered >@@ -328,11 +332,15 @@ RECORD: while ( ) { > next; > } > if ($record) { >+ > # transcode the record to UTF8 if needed & applicable. >- if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { >- my ($guessed_charset, $charset_errors); >- ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : '')); >- if ($guessed_charset eq 'failed') { >+ if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) { >+ my ( $guessed_charset, $charset_errors ); >+ ( $record, $guessed_charset, $charset_errors ) = MarcToUTF8Record( >+ $record, >+ $marc_flavour . ( ( $authorities and $marc_flavour ne "MARC21" ) ? 'AUTH' : '' ) >+ ); >+ if ( $guessed_charset eq 'failed' ) { > warn "ERROR: failed to perform character conversion for record $record_number\n"; > next RECORD; > } >@@ -340,8 +348,7 @@ RECORD: while ( ) { > SetUTF8Flag($record); > &$localcust($record) if $localcust; > push @{$marc_records}, $record; >- } >- else { >+ } else { > last; > } > } >@@ -349,55 +356,60 @@ RECORD: while ( ) { > $record_number = 0; > my $records_total = @{$marc_records}; > $schema->txn_begin; >-RECORD: foreach my $record (@{$marc_records}) { >+RECORD: foreach my $record ( @{$marc_records} ) { > $record_number++; >- if (($verbose//1) == 1) { #no dot for verbose==2 >- print "." . ($record_number % 100 == 0 ? "\n$record_number" : ''); >+ if ( ( $verbose // 1 ) == 1 ) { #no dot for verbose==2 >+ print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' ); > } > >- if ($marc_mod_template_id > 0) { >+ if ( $marc_mod_template_id > 0 ) { > print "Modifying MARC\n" if $verbose; > ModifyRecordWithTemplate( $marc_mod_template_id, $record ); > } > > my $isbn; >+ > # remove trailing - in isbn (only for biblios, of course) >- if ($biblios && ($cleanisbn || $isbn_check)) { >- my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; >+ if ( $biblios && ( $cleanisbn || $isbn_check ) ) { >+ my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; > my $field = $record->field($tag); > $isbn = $field && $field->subfield('a'); >- if ($isbn && $cleanisbn) { >+ if ( $isbn && $cleanisbn ) { > $isbn =~ s/-//g; >- $field->update('a' => $isbn); >+ $field->update( 'a' => $isbn ); > } > } >+ > # search for duplicates (based on Local-number) >- my $originalid = GetRecordId($record, $tagid, $subfieldid); >+ my $originalid = GetRecordId( $record, $tagid, $subfieldid ); > my $matched_record_id = undef; > if ($match) { > require C4::Search; >- my $server = ($authorities ? 'authorityserver' : 'biblioserver'); >- my $query = build_query($match, $record); >+ my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); >+ my $query = build_query( $match, $record ); > $logger->debug("Bulkmarcimport: $query"); >- my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]); >+ my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); >+ > # changed to warn so able to continue with one broken record >- if (defined $error) { >+ if ( defined $error ) { > warn "unable to search the database for duplicates : $error"; >- printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile); >+ printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile); > next RECORD; > } > $logger->debug("Bulkmarcimport: $query $server : $totalhits"); >+ > # sub SimpleSearch could return undefined, but only on error, so > # should not really need to safeguard here, but do so anyway > $results //= []; >- if (@{$results} == 1) { >- my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]); >+ if ( @{$results} == 1 ) { >+ my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] ); > SetUTF8Flag($matched_record); >- $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid); >+ $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid ); >+ >+ if ( $authorities && $marc_flavour ) { > >- if ($authorities && $marc_flavour) { > #Skip if authority in database is the same or newer than the incoming record >- if (RecordRevisionIsGtOrEq($matched_record, $record)) { >+ if ( RecordRevisionIsGtOrEq( $matched_record, $record ) ) { > if ($yamlfile) { > $yamlhash->{$originalid} = YAMLFileEntry( > $matched_record, >@@ -408,257 +420,279 @@ RECORD: foreach my $record (@{$marc_records}) { > next; > } > } >- } >- elsif(@{$results} > 1) { >+ } elsif ( @{$results} > 1 ) { > $logger->debug("More than one match for: $query"); >- } >- else { >+ } else { > $logger->debug("No match for: $query"); > } > >- if ($keepids && $originalid) { >+ if ( $keepids && $originalid ) { > my $storeidfield; >- if (length($keepids) == 3) { >- $storeidfield = MARC::Field->new($keepids, $originalid); >+ if ( length($keepids) == 3 ) { >+ $storeidfield = MARC::Field->new( $keepids, $originalid ); > } else { >- $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid); >+ $storeidfield = >+ MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); > } > $record->insert_fields_ordered($storeidfield); >- $record->delete_field($record->field($tagid)); >+ $record->delete_field( $record->field($tagid) ); > } > } > > foreach my $stringfilter (@$filters) { >- if (length($stringfilter) == 3) { >- foreach my $field ($record->field($stringfilter)) { >+ if ( length($stringfilter) == 3 ) { >+ foreach my $field ( $record->field($stringfilter) ) { > $record->delete_field($field); >- $logger->debug("Removed: ", $field->as_string); >+ $logger->debug( "Removed: ", $field->as_string ); > } >- } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { >- my $removetag = $1; >+ } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) { >+ my $removetag = $1; > my $removesubfield = $2; >- my $removematch = $3; >- if (($removetag > "010") && $removesubfield) { >- foreach my $field ($record->field($removetag)) { >- $field->delete_subfield(code => "$removesubfield", match => $removematch); >- $logger->debug("Potentially removed: ", $field->subfield($removesubfield)); >+ my $removematch = $3; >+ if ( ( $removetag > "010" ) && $removesubfield ) { >+ foreach my $field ( $record->field($removetag) ) { >+ $field->delete_subfield( code => "$removesubfield", match => $removematch ); >+ $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) ); > } > } > } > } > unless ($test_parameter) { > if ($authorities) { >- my $authtypecode = GuessAuthTypeCode($record, $heading_fields); >+ my $authtypecode = GuessAuthTypeCode( $record, $heading_fields ); > my $authid; > > if ($matched_record_id) { > if ($update) { > ## Authority has an id and is in database: update >- eval { ($authid) = ModAuthority($matched_record_id, $record, $authtypecode) }; >+ eval { ($authid) = ModAuthority( $matched_record_id, $record, $authtypecode ) }; > if ($@) { > warn "ERROR: Update authority $matched_record_id failed: $@\n"; >- printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if ($logfile); >- } >- else { >- printlog({ id => $authid, op => "update", status => "ok" }) if ($logfile); >+ printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); >+ } else { >+ printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile); > } >- } >- elsif ($logfile) { >+ } elsif ($logfile) { > warn "WARNING: Update authority $originalid skipped"; >- printlog({ >- id => $matched_record_id, >- op => "update", >- status => "warning: authority already in database and option -update not enabled, skipping..." >- }); >+ printlog( >+ { >+ id => $matched_record_id, >+ op => "update", >+ status => >+ "warning: authority already in database and option -update not enabled, skipping..." >+ } >+ ); > } >- } >- elsif ($insert) { >+ } elsif ($insert) { > ## An authid is defined but no authority in database: insert >- eval { ($authid) = AddAuthority($record, undef, $authtypecode) }; >+ eval { ($authid) = AddAuthority( $record, undef, $authtypecode ) }; > if ($@) { > warn "ERROR: Insert authority $originalid failed: $@\n"; >- printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile); >- } >- else { >- printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile); >+ printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); >+ } else { >+ printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile); > } >- } >- else { >+ } else { > warn "WARNING: Insert authority $originalid skipped"; >- printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile); >+ printlog( >+ { >+ id => $originalid, op => "insert", >+ status => "warning : biblio not in database and option -insert not enabled, skipping..." >+ } >+ ) if ($logfile); > } > > if ($yamlfile) { > $yamlhash->{$originalid} = YAMLFileEntry( > $record, > $authid, >- 1 #@FIXME: Really always updated? >+ 1 #@FIXME: Really always updated? > ); > } >- } >- else { >- my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id); >+ } else { >+ my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id ); >+ > # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter >- if (!$matched_record_id && $isbn_check && $isbn) { >+ if ( !$matched_record_id && $isbn_check && $isbn ) { > $sth_isbn->execute($isbn); >- ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow; >+ ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow; > } > >- if (defined $idmapfl && $matched_record_id) { >- if ($sourcetag < "010") { >- if ($record->field($sourcetag)) { >+ if ( defined $idmapfl && $matched_record_id ) { >+ if ( $sourcetag < "010" ) { >+ if ( $record->field($sourcetag) ) { > my $source = $record->field($sourcetag)->data(); >- printf($idmapfh "%s|%s\n", $source, $matched_record_id); >+ printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); > } >- } >- else { >- my $source = $record->subfield($sourcetag, $sourcesubfield); >- printf($idmapfh "%s|%s\n", $source, $matched_record_id); >+ } else { >+ my $source = $record->subfield( $sourcetag, $sourcesubfield ); >+ printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); > } > } > > # Create biblio, unless we already have it (either match or ISBN) > if ($matched_record_id) { >- eval{ >- $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber; >- }; >+ eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; }; > if ($update) { > my $success; >- eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $mod_biblio_options) }; >+ eval { >+ $success = ModBiblio( >+ $record, $matched_record_id, GetFrameworkCode($matched_record_id), >+ $mod_biblio_options >+ ); >+ }; > if ($@) { > warn "ERROR: Update biblio $matched_record_id failed: $@\n"; > printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); > next RECORD; >- } >- elsif (!$success) { >+ } elsif ( !$success ) { > warn "ERROR: Update biblio $matched_record_id failed for unknown reason"; > printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); > next RECORD; >- } >- else { >+ } else { > $record_id = $matched_record_id; > printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile); > } >- } >- else { >+ } else { > warn "WARNING: Update biblio $originalid skipped"; >- printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile); >+ printlog( >+ { >+ id => $matched_record_id, op => "update", >+ status => "warning : already in database and option -update not enabled, skipping..." >+ } >+ ) if ($logfile); > } >- } >- elsif ($insert) { >+ } elsif ($insert) { > my $record_clone = $record->clone(); > C4::Biblio::_strip_item_fields($record_clone); >- eval { ($record_id, $biblioitemnumber) = AddBiblio($record_clone, $framework, $add_biblio_options) }; >+ eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) }; > if ($@) { > warn "ERROR: Insert biblio $originalid failed: $@\n"; > printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); > next RECORD; >- } >- else { >+ } else { > printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile); > } >+ > # If incoming record has bib ids set we need to transfer > # new ids from record_clone to incoming record to avoid > # working on wrong record (the original record) later on > # when adding items for example >- C4::Biblio::_koha_marc_update_bib_ids($record, $framework, $record_id, $biblioitemnumber); >- } >- else { >+ C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber ); >+ } else { > warn "WARNING: Insert biblio $originalid skipped"; >- printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile); >+ printlog( >+ { >+ id => $originalid, op => "insert", >+ status => "warning : biblio not in database and option -insert not enabled, skipping..." >+ } >+ ) if ($logfile); > next RECORD; > } > my $record_has_added_items = 0; > if ($record_id) { > $yamlhash->{$originalid} = $record_id if $yamlfile; >- eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); }; >+ eval { >+ ( $itemnumbers_ref, $errors_ref ) = >+ AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework ); >+ }; > my $error_adding = $@; > $record_has_added_items = @{$itemnumbers_ref}; > > if ($error_adding) { > warn "ERROR: Adding items to bib $record_id failed: $error_adding"; >- printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile); >+ printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); >+ > # if we failed because of an exception, assume that > # the MARC columns in biblioitems were not set. > next RECORD; > } >- if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { >+ if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } >+ @$errors_ref ) >+ { > # Find the record called 'barcode' >- my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode'); >+ my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode'); >+ > # Now remove any items that didn't have a duplicate_barcode error, > # erase the barcodes on items that did, and re-add those items. > my %dupes; >- foreach my $i (0 .. $#{$errors_ref}) { >+ foreach my $i ( 0 .. $#{$errors_ref} ) { > my $ref = $errors_ref->[$i]; >- if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { >- $dupes{$ref->{item_sequence}} = 1; >+ if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) { >+ $dupes{ $ref->{item_sequence} } = 1; >+ > # Delete the error message because we're going to > # retry this one. > delete $errors_ref->[$i]; > } > } > my $seq = 0; >- foreach my $field ($record->field($tag)) { >+ foreach my $field ( $record->field($tag) ) { > $seq++; >- if ($dupes{$seq}) { >+ if ( $dupes{$seq} ) { >+ > # Here we remove the barcode >- $field->delete_subfield(code => $sub); >- } >- else { >+ $field->delete_subfield( code => $sub ); >+ } else { >+ > # otherwise we delete the field because we don't want > # two of them > $record->delete_fields($field); > } > } >+ > # Now re-add the record as before, adding errors to the prev list > my $more_errors; >- eval { ($itemnumbers_ref, $more_errors) = >- AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); }; >+ eval { >+ ( $itemnumbers_ref, $more_errors ) = >+ AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' ); >+ }; > if ($@) { > warn "ERROR: Adding items to bib $record_id failed: $@\n"; >- printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile); >+ printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); >+ > # if we failed because of an exception, assume that > # the MARC columns in biblioitems were not set. > next RECORD; > } > $record_has_added_items ||= @{$itemnumbers_ref}; >- if (@{$more_errors}) { >+ if ( @{$more_errors} ) { > push @$errors_ref, @{$more_errors}; > } > } > >- if ($record_has_added_items) { >- printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile); >+ if ($record_has_added_items) { >+ printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile); > } > >- if (@{$errors_ref}) { >- report_item_errors($record_id, $errors_ref); >+ if ( @{$errors_ref} ) { >+ report_item_errors( $record_id, $errors_ref ); > } > > my $biblio = Koha::Biblios->find($record_id); > $record = $biblio->metadata->record( { embed_items => 1 } ); > > push @search_engine_record_ids, $record_id; >- push @search_engine_records, $record; >+ push @search_engine_records, $record; > } > } >- if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) { >+ if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) { > $schema->txn_commit; > $schema->txn_begin; > if ($indexer) { >- $indexer->update_index(\@search_engine_record_ids, \@search_engine_records); >- if (C4::Context->preference('AutoLinkBiblios')) { >+ $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ); >+ if ( C4::Context->preference('AutoLinkBiblios') ) { > foreach my $record (@search_engine_records) { >- BiblioAutoLink($record, $framework); >+ BiblioAutoLink( $record, $framework ); > } > } > @search_engine_record_ids = (); >- @search_engine_records = (); >+ @search_engine_records = (); > } > } > } >- print $record->as_formatted() . "\n" if ($verbose//0) == 2; >- last if $record_number == $number; >+ print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2; >+ last if $record_number == $number; > } > $schema->txn_commit; > >@@ -679,104 +713,112 @@ if ($logfile) { > } > if ($yamlfile) { > open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; >- print $yamlfileout Encode::decode_utf8(YAML::XS::Dump($yamlhash)); >+ print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) ); > } > exit 0; > > sub YAMLFileEntry { >- my ($record, $record_id, $updated) = @_; >+ my ( $record, $record_id, $updated ) = @_; > >- my $entry = { >- authid => $record_id >- }; >+ my $entry = { authid => $record_id }; > > # we recover all subfields of the heading authorities > my @subfields; >- foreach my $field ($record->field("2..")) { >+ foreach my $field ( $record->field("2..") ) { > push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); > } > $entry->{'subfields'} = \@subfields; >- $entry->{'updated'} = $updated; >+ $entry->{'updated'} = $updated; > > return $entry; > } > > sub RecordRevisionIsGtOrEq { >- my ($record_a, $record_b) = @_; >- return $record_a->field('005') && $record_b->field('005') && >- $record_a->field('005')->data && $record_b->field('005')->data && >- $record_a->field('005')->data >= $record_b->field('005')->data; >+ my ( $record_a, $record_b ) = @_; >+ return >+ $record_a->field('005') >+ && $record_b->field('005') >+ && $record_a->field('005')->data >+ && $record_b->field('005')->data >+ && $record_a->field('005')->data >= $record_b->field('005')->data; > } > > sub GetRecordId { > my $marcrecord = shift; >- my $tag = shift; >- my $subfield = shift; >- if ($tag lt "010") { >+ my $tag = shift; >+ my $subfield = shift; >+ if ( $tag lt "010" ) { > return $marcrecord->field($tag)->data() if $marcrecord->field($tag); >- } >- elsif ($subfield) { >- if ($marcrecord->field($tag)) { >- return $marcrecord->subfield($tag, $subfield); >+ } elsif ($subfield) { >+ if ( $marcrecord->field($tag) ) { >+ return $marcrecord->subfield( $tag, $subfield ); > } > } > } >+ > sub build_query { >- my ($match, $record) = @_; >+ my ( $match, $record ) = @_; > my @searchstrings; > > foreach my $matchpoint (@$match) { >- my $query = build_simplequery($matchpoint, $record); >- push (@searchstrings, $query) if $query; >+ my $query = build_simplequery( $matchpoint, $record ); >+ push( @searchstrings, $query ) if $query; > } > my $op = 'AND'; >- return join(" $op ", @searchstrings); >+ return join( " $op ", @searchstrings ); > } >+ > sub build_simplequery { >- my ($matchpoint, $record) = @_; >+ my ( $matchpoint, $record ) = @_; > > my @searchstrings; >- my ($index, $record_data) = split (/,/, $matchpoint); >- if ($record_data =~ /(\d{3})(.*)/) { >- my ($tag, $subfields) = ($1, $2); >- foreach my $field ($record->field($tag)) { >- if (length($field->as_string("$subfields")) > 0) { >- push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\""); >+ my ( $index, $record_data ) = split( /,/, $matchpoint ); >+ if ( $record_data =~ /(\d{3})(.*)/ ) { >+ my ( $tag, $subfields ) = ( $1, $2 ); >+ foreach my $field ( $record->field($tag) ) { >+ if ( length( $field->as_string("$subfields") ) > 0 ) { >+ push( @searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"" ); > } > } >- } >- else { >+ } else { > print "Invalid matchpoint format, invalid marc-field: $matchpoint\n"; > } > my $op = 'AND'; >- return join(" $op ", @searchstrings); >+ return join( " $op ", @searchstrings ); > } >+ > sub report_item_errors { > my $biblionumber = shift; >- my $errors_ref = shift; >+ my $errors_ref = shift; > >- foreach my $error (@{ $errors_ref }) { >+ foreach my $error ( @{$errors_ref} ) { > next if !$error; >- my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): "; >+ my $msg = >+ "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): "; > my $error_code = $error->{'error_code'}; > $error_code =~ s/_/ /g; > $msg .= "$error_code $error->{'error_information'}"; > print $msg, "\n"; > } > } >+ > sub printlog { > my $logelements = shift; >- print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n"; >+ print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; > } >+ > sub get_heading_fields { > my $headingfields; > if ($authtypes) { > $headingfields = YAML::XS::LoadFile($authtypes); > $headingfields = { C4::Context->preference('marcflavour') => $headingfields }; >- $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); >+ $logger->debug( Encode::decode_utf8( YAML::XS::Dump($headingfields) ) ); > } > unless ($headingfields) { >- $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); >+ $headingfields = $dbh->selectall_hashref( >+ "SELECT auth_tag_to_report, authtypecode from auth_types", 'auth_tag_to_report', >+ { Slice => {} } >+ ); > $headingfields = { C4::Context->preference('marcflavour') => $headingfields }; > } > return $headingfields; >-- >2.43.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 30349
:
132127
|
132128
|
132130
|
160492
|
160493
|
160494
|
160913
|
160914
|
160915
|
160916