@@ -, +, @@
---
misc/migration_tools/bulkmarcimport.pl | 564 +++++++++++++------------
1 file changed, 303 insertions(+), 261 deletions(-)
--- a/misc/migration_tools/bulkmarcimport.pl
+++ a/misc/migration_tools/bulkmarcimport.pl
@@ -2,6 +2,7 @@
# Import an iso2709 file into Koha 3
use Modern::Perl;
+
#use diagnostics;
# Koha modules used
@@ -21,8 +22,8 @@ use C4::Biblio qw(
BiblioAutoLink
);
use C4::Koha;
-use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
-use C4::Items qw( AddItemBatchFromMarc );
+use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
+use C4::Items qw( AddItemBatchFromMarc );
use C4::MarcModificationTemplates qw(
GetModificationTemplates
ModifyRecordWithTemplate
@@ -30,11 +31,11 @@ use C4::MarcModificationTemplates qw(
use C4::AuthoritiesMarc qw( GuessAuthTypeCode GuessAuthId GetAuthority ModAuthority AddAuthority );
use YAML::XS;
-use Time::HiRes qw( gettimeofday );
+use Time::HiRes qw( gettimeofday );
use Getopt::Long qw( GetOptions );
use IO::File;
use Pod::Usage qw( pod2usage );
-use FindBin ();
+use FindBin ();
use Koha::Logger;
use Koha::Biblios;
@@ -42,8 +43,8 @@ use Koha::SearchEngine;
use Koha::SearchEngine::Search;
use open qw( :std :encoding(UTF-8) );
-binmode(STDOUT, ":encoding(UTF-8)");
-my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
+binmode( STDOUT, ":encoding(UTF-8)" );
+my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 );
my $version;
my $delete;
my $test_parameter;
@@ -72,43 +73,43 @@ my $idmapfl;
my $dedup_barcode;
my $framework = '';
my $localcust;
-my $marc_mod_template = '';
+my $marc_mod_template = '';
my $marc_mod_template_id = -1;
$| = 1;
GetOptions(
- 'commit:f' => \$commit,
- 'file:s' => \$input_marc_file,
- 'n:f' => \$number,
- 'o|offset:f' => \$offset,
- 'h' => \$version,
- 'd' => \$delete,
- 't|test' => \$test_parameter,
- 's' => \$skip_marc8_conversion,
- 'c:s' => \$char_encoding,
- 'v:+' => \$verbose,
- 'fk' => \$fk_off,
- 'm:s' => \$format,
- 'l:s' => \$logfile,
- 'append' => \$append,
- 'k|keepids:s' => \$keepids,
- 'b|biblios' => \$biblios,
- 'a|authorities' => \$authorities,
- 'authtypes:s' => \$authtypes,
- 'filter=s@' => \$filters,
- 'insert' => \$insert,
- 'update' => \$update,
- 'all' => \$all,
- 'match=s@' => \$match,
- 'i|isbn' => \$isbn_check,
- 'x:s' => \$sourcetag,
- 'y:s' => \$sourcesubfield,
- 'idmap:s' => \$idmapfl,
- 'cleanisbn!' => \$cleanisbn,
- 'yaml:s' => \$yamlfile,
- 'dedupbarcode' => \$dedup_barcode,
- 'framework=s' => \$framework,
- 'custom:s' => \$localcust,
+ 'commit:f' => \$commit,
+ 'file:s' => \$input_marc_file,
+ 'n:f' => \$number,
+ 'o|offset:f' => \$offset,
+ 'h' => \$version,
+ 'd' => \$delete,
+ 't|test' => \$test_parameter,
+ 's' => \$skip_marc8_conversion,
+ 'c:s' => \$char_encoding,
+ 'v:+' => \$verbose,
+ 'fk' => \$fk_off,
+ 'm:s' => \$format,
+ 'l:s' => \$logfile,
+ 'append' => \$append,
+ 'k|keepids:s' => \$keepids,
+ 'b|biblios' => \$biblios,
+ 'a|authorities' => \$authorities,
+ 'authtypes:s' => \$authtypes,
+ 'filter=s@' => \$filters,
+ 'insert' => \$insert,
+ 'update' => \$update,
+ 'all' => \$all,
+ 'match=s@' => \$match,
+ 'i|isbn' => \$isbn_check,
+ 'x:s' => \$sourcetag,
+ 'y:s' => \$sourcesubfield,
+ 'idmap:s' => \$idmapfl,
+ 'cleanisbn!' => \$cleanisbn,
+ 'yaml:s' => \$yamlfile,
+ 'dedupbarcode' => \$dedup_barcode,
+ 'framework=s' => \$framework,
+ 'custom:s' => \$localcust,
'marcmodtemplate:s' => \$marc_mod_template,
);
@@ -116,21 +117,22 @@ $biblios ||= !$authorities;
$insert ||= !$update;
my $writemode = ($append) ? "a" : "w";
-pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities;
+pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval )
+ if $biblios && $authorities;
if ($all) {
$insert = 1;
$update = 1;
}
-my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
-my $mod_biblio_options = {
- disable_autolink => $using_elastic_search,
+my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' );
+my $mod_biblio_options = {
+ disable_autolink => $using_elastic_search,
skip_record_index => $using_elastic_search,
- overlay_context => { source => 'bulkmarcimport' }
+ overlay_context => { source => 'bulkmarcimport' }
};
my $add_biblio_options = {
- disable_autolink => $using_elastic_search,
+ disable_autolink => $using_elastic_search,
skip_record_index => $using_elastic_search
};
@@ -140,70 +142,70 @@ my $indexer;
if ($using_elastic_search) {
use Koha::SearchEngine::Elasticsearch::Indexer;
$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
- { index => $authorities ?
- $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
- $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
+ {
+ index => $authorities
+ ? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX
+ : $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
}
);
}
-if ($version || ($input_marc_file eq '')) {
+if ( $version || ( $input_marc_file eq '' ) ) {
pod2usage( -verbose => 2 );
exit;
}
-if( $update && !( $match || $isbn_check ) ) {
+if ( $update && !( $match || $isbn_check ) ) {
warn "Using -update without -match or -isbn seems to be useless.\n";
}
-if (defined $localcust) { #local customize module
- if (!-e $localcust) {
- $localcust = $localcust || 'LocalChanges'; #default name
- $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
- $localcust =~ s/\.pm$//; #remove extension
- my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
- if (-e $fqcust) {
- $localcust= $fqcust;
- }
- else {
+if ( defined $localcust ) { #local customize module
+ if ( !-e $localcust ) {
+ $localcust = $localcust || 'LocalChanges'; #default name
+ $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
+ $localcust =~ s/\.pm$//; #remove extension
+ my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
+ if ( -e $fqcust ) {
+ $localcust = $fqcust;
+ } else {
print "WARNING: customize module $localcust.pm not found!\n";
exit 1;
}
}
- require $localcust if $localcust;
+ require $localcust if $localcust;
$localcust = \&customize if $localcust;
}
-if($marc_mod_template ne '') {
+if ( $marc_mod_template ne '' ) {
my @templates = GetModificationTemplates();
foreach my $this_template (@templates) {
- if($this_template->{'name'} eq $marc_mod_template) {
- if($marc_mod_template_id < 0) {
+ if ( $this_template->{'name'} eq $marc_mod_template ) {
+ if ( $marc_mod_template_id < 0 ) {
$marc_mod_template_id = $this_template->{'template_id'};
} else {
- print "WARNING: MARC modification template name " .
- "'$marc_mod_template' matches multiple templates. " .
- "Please rename these templates\n";
+ print "WARNING: MARC modification template name "
+ . "'$marc_mod_template' matches multiple templates. "
+ . "Please rename these templates\n";
exit 1;
}
}
}
- if($marc_mod_template_id < 0) {
+ if ( $marc_mod_template_id < 0 ) {
die "Can't located MARC modification template '$marc_mod_template'\n";
} else {
print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose;
}
}
-my $dbh = C4::Context->dbh;
+my $dbh = C4::Context->dbh;
my $heading_fields = get_heading_fields();
my $idmapfh;
-if (defined $idmapfl) {
- open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
+if ( defined $idmapfl ) {
+ open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n";
}
-if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
- $sourcetag = "910";
+if ( ( not defined $sourcesubfield ) && ( not defined $sourcetag ) ) {
+ $sourcetag = "910";
$sourcesubfield = "a";
}
@@ -225,8 +227,7 @@ if ($delete) {
$dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
$dbh->do("DELETE FROM items");
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
- }
- else {
+ } else {
print "Deleting authorities\n";
$dbh->do("truncate auth_header");
}
@@ -244,7 +245,7 @@ my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
my $searcher = Koha::SearchEngine::Search->new(
{
index => (
- $authorities
+ $authorities
? $Koha::SearchEngine::AUTHORITIES_INDEX
: $Koha::SearchEngine::BIBLIOS_INDEX
)
@@ -254,8 +255,9 @@ my $searcher = Koha::SearchEngine::Search->new(
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
my $starttime = gettimeofday;
-my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
-if (defined $format && $format =~ /XML/i) {
+my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
+if ( defined $format && $format =~ /XML/i ) {
+
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
# appears to try to convert incoming XML records from MARC-8
# to UTF-8. Setting the BinaryEncoding key turns that off
@@ -266,14 +268,14 @@ if (defined $format && $format =~ /XML/i) {
# extract the records, not using regexes to look
# for .*.
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
- my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
+ my $recordformat = ( $marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour) );
+
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
- $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
+ $recordformat = $recordformat . "AUTH" if ( $authorities and $marc_flavour ne "MARC21" );
$MARC::File::XML::_load_args{RecordFormat} = $recordformat;
- $batch = MARC::Batch->new('XML', $fh);
-}
-else {
- $batch = MARC::Batch->new('USMARC', $fh);
+ $batch = MARC::Batch->new( 'XML', $fh );
+} else {
+ $batch = MARC::Batch->new( 'USMARC', $fh );
}
$batch->warnings_off();
@@ -284,19 +286,19 @@ my $yamlhash;
# Skip file offset
if ($offset) {
print "Skipping file offset: $offset records\n";
- $batch->next() while ($offset--);
+ $batch->next() while ( $offset-- );
}
-my ($tagid, $subfieldid);
+my ( $tagid, $subfieldid );
if ($authorities) {
$tagid = '001';
-}
-else {
- ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
+} else {
+ ( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber");
$tagid ||= "001";
}
my $sth_isbn;
+
# the SQL query to search on isbn
if ($isbn_check) {
$sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
@@ -304,21 +306,23 @@ if ($isbn_check) {
my $loghandle;
if ($logfile) {
- $loghandle= IO::File->new($logfile, $writemode);
+ $loghandle = IO::File->new( $logfile, $writemode );
print $loghandle "id;operation;status\n";
}
my $record_number = 0;
-my $logger = Koha::Logger->get;
-my $schema = Koha::Database->schema;
-my $marc_records = [];
-RECORD: while ( ) {
+my $logger = Koha::Logger->get;
+my $schema = Koha::Database->schema;
+my $marc_records = [];
+RECORD: while () {
my $record;
$record_number++;
+
# get record
eval { $record = $batch->next() };
if ($@) {
print "Bad MARC record $record_number: $@ skipped\n";
+
# FIXME - because MARC::Batch->next() combines grabbing the next
# blob and parsing it into one operation, a correctable condition
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered
@@ -328,11 +332,15 @@ RECORD: while ( ) {
next;
}
if ($record) {
+
# transcode the record to UTF8 if needed & applicable.
- if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
- my ($guessed_charset, $charset_errors);
- ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
- if ($guessed_charset eq 'failed') {
+ if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) {
+ my ( $guessed_charset, $charset_errors );
+ ( $record, $guessed_charset, $charset_errors ) = MarcToUTF8Record(
+ $record,
+ $marc_flavour . ( ( $authorities and $marc_flavour ne "MARC21" ) ? 'AUTH' : '' )
+ );
+ if ( $guessed_charset eq 'failed' ) {
warn "ERROR: failed to perform character conversion for record $record_number\n";
next RECORD;
}
@@ -340,8 +348,7 @@ RECORD: while ( ) {
SetUTF8Flag($record);
&$localcust($record) if $localcust;
push @{$marc_records}, $record;
- }
- else {
+ } else {
last;
}
}
@@ -349,55 +356,60 @@ RECORD: while ( ) {
$record_number = 0;
my $records_total = @{$marc_records};
$schema->txn_begin;
-RECORD: foreach my $record (@{$marc_records}) {
+RECORD: foreach my $record ( @{$marc_records} ) {
$record_number++;
- if (($verbose//1) == 1) { #no dot for verbose==2
- print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
+ if ( ( $verbose // 1 ) == 1 ) { #no dot for verbose==2
+ print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' );
}
- if ($marc_mod_template_id > 0) {
+ if ( $marc_mod_template_id > 0 ) {
print "Modifying MARC\n" if $verbose;
ModifyRecordWithTemplate( $marc_mod_template_id, $record );
}
my $isbn;
+
# remove trailing - in isbn (only for biblios, of course)
- if ($biblios && ($cleanisbn || $isbn_check)) {
- my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
+ if ( $biblios && ( $cleanisbn || $isbn_check ) ) {
+ my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
my $field = $record->field($tag);
$isbn = $field && $field->subfield('a');
- if ($isbn && $cleanisbn) {
+ if ( $isbn && $cleanisbn ) {
$isbn =~ s/-//g;
- $field->update('a' => $isbn);
+ $field->update( 'a' => $isbn );
}
}
+
# search for duplicates (based on Local-number)
- my $originalid = GetRecordId($record, $tagid, $subfieldid);
+ my $originalid = GetRecordId( $record, $tagid, $subfieldid );
my $matched_record_id = undef;
if ($match) {
require C4::Search;
- my $server = ($authorities ? 'authorityserver' : 'biblioserver');
- my $query = build_query($match, $record);
+ my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
+ my $query = build_query( $match, $record );
$logger->debug("Bulkmarcimport: $query");
- my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
+ my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
+
# changed to warn so able to continue with one broken record
- if (defined $error) {
+ if ( defined $error ) {
warn "unable to search the database for duplicates : $error";
- printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
+ printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile);
next RECORD;
}
$logger->debug("Bulkmarcimport: $query $server : $totalhits");
+
# sub SimpleSearch could return undefined, but only on error, so
# should not really need to safeguard here, but do so anyway
$results //= [];
- if (@{$results} == 1) {
- my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
+ if ( @{$results} == 1 ) {
+ my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] );
SetUTF8Flag($matched_record);
- $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
+ $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid );
+
+ if ( $authorities && $marc_flavour ) {
- if ($authorities && $marc_flavour) {
#Skip if authority in database is the same or newer than the incoming record
- if (RecordRevisionIsGtOrEq($matched_record, $record)) {
+ if ( RecordRevisionIsGtOrEq( $matched_record, $record ) ) {
if ($yamlfile) {
$yamlhash->{$originalid} = YAMLFileEntry(
$matched_record,
@@ -408,257 +420,279 @@ RECORD: foreach my $record (@{$marc_records}) {
next;
}
}
- }
- elsif(@{$results} > 1) {
+ } elsif ( @{$results} > 1 ) {
$logger->debug("More than one match for: $query");
- }
- else {
+ } else {
$logger->debug("No match for: $query");
}
- if ($keepids && $originalid) {
+ if ( $keepids && $originalid ) {
my $storeidfield;
- if (length($keepids) == 3) {
- $storeidfield = MARC::Field->new($keepids, $originalid);
+ if ( length($keepids) == 3 ) {
+ $storeidfield = MARC::Field->new( $keepids, $originalid );
} else {
- $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
+ $storeidfield =
+ MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
}
$record->insert_fields_ordered($storeidfield);
- $record->delete_field($record->field($tagid));
+ $record->delete_field( $record->field($tagid) );
}
}
foreach my $stringfilter (@$filters) {
- if (length($stringfilter) == 3) {
- foreach my $field ($record->field($stringfilter)) {
+ if ( length($stringfilter) == 3 ) {
+ foreach my $field ( $record->field($stringfilter) ) {
$record->delete_field($field);
- $logger->debug("Removed: ", $field->as_string);
+ $logger->debug( "Removed: ", $field->as_string );
}
- } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
- my $removetag = $1;
+ } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) {
+ my $removetag = $1;
my $removesubfield = $2;
- my $removematch = $3;
- if (($removetag > "010") && $removesubfield) {
- foreach my $field ($record->field($removetag)) {
- $field->delete_subfield(code => "$removesubfield", match => $removematch);
- $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
+ my $removematch = $3;
+ if ( ( $removetag > "010" ) && $removesubfield ) {
+ foreach my $field ( $record->field($removetag) ) {
+ $field->delete_subfield( code => "$removesubfield", match => $removematch );
+ $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) );
}
}
}
}
unless ($test_parameter) {
if ($authorities) {
- my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
+ my $authtypecode = GuessAuthTypeCode( $record, $heading_fields );
my $authid;
if ($matched_record_id) {
if ($update) {
## Authority has an id and is in database: update
- eval { ($authid) = ModAuthority($matched_record_id, $record, $authtypecode) };
+ eval { ($authid) = ModAuthority( $matched_record_id, $record, $authtypecode ) };
if ($@) {
warn "ERROR: Update authority $matched_record_id failed: $@\n";
- printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if ($logfile);
- }
- else {
- printlog({ id => $authid, op => "update", status => "ok" }) if ($logfile);
+ printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
+ } else {
+ printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile);
}
- }
- elsif ($logfile) {
+ } elsif ($logfile) {
warn "WARNING: Update authority $originalid skipped";
- printlog({
- id => $matched_record_id,
- op => "update",
- status => "warning: authority already in database and option -update not enabled, skipping..."
- });
+ printlog(
+ {
+ id => $matched_record_id,
+ op => "update",
+ status =>
+ "warning: authority already in database and option -update not enabled, skipping..."
+ }
+ );
}
- }
- elsif ($insert) {
+ } elsif ($insert) {
## An authid is defined but no authority in database: insert
- eval { ($authid) = AddAuthority($record, undef, $authtypecode) };
+ eval { ($authid) = AddAuthority( $record, undef, $authtypecode ) };
if ($@) {
warn "ERROR: Insert authority $originalid failed: $@\n";
- printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
- }
- else {
- printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
+ printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
+ } else {
+ printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile);
}
- }
- else {
+ } else {
warn "WARNING: Insert authority $originalid skipped";
- printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile);
+ printlog(
+ {
+ id => $originalid, op => "insert",
+ status => "warning : biblio not in database and option -insert not enabled, skipping..."
+ }
+ ) if ($logfile);
}
if ($yamlfile) {
$yamlhash->{$originalid} = YAMLFileEntry(
$record,
$authid,
- 1 #@FIXME: Really always updated?
+ 1 #@FIXME: Really always updated?
);
}
- }
- else {
- my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
+ } else {
+ my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id );
+
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
- if (!$matched_record_id && $isbn_check && $isbn) {
+ if ( !$matched_record_id && $isbn_check && $isbn ) {
$sth_isbn->execute($isbn);
- ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
+ ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow;
}
- if (defined $idmapfl && $matched_record_id) {
- if ($sourcetag < "010") {
- if ($record->field($sourcetag)) {
+ if ( defined $idmapfl && $matched_record_id ) {
+ if ( $sourcetag < "010" ) {
+ if ( $record->field($sourcetag) ) {
my $source = $record->field($sourcetag)->data();
- printf($idmapfh "%s|%s\n", $source, $matched_record_id);
+ printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
}
- }
- else {
- my $source = $record->subfield($sourcetag, $sourcesubfield);
- printf($idmapfh "%s|%s\n", $source, $matched_record_id);
+ } else {
+ my $source = $record->subfield( $sourcetag, $sourcesubfield );
+ printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
}
}
# Create biblio, unless we already have it (either match or ISBN)
if ($matched_record_id) {
- eval{
- $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
- };
+ eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; };
if ($update) {
my $success;
- eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $mod_biblio_options) };
+ eval {
+ $success = ModBiblio(
+ $record, $matched_record_id, GetFrameworkCode($matched_record_id),
+ $mod_biblio_options
+ );
+ };
if ($@) {
warn "ERROR: Update biblio $matched_record_id failed: $@\n";
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
next RECORD;
- }
- elsif (!$success) {
+ } elsif ( !$success ) {
warn "ERROR: Update biblio $matched_record_id failed for unknown reason";
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
next RECORD;
- }
- else {
+ } else {
$record_id = $matched_record_id;
printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
}
- }
- else {
+ } else {
warn "WARNING: Update biblio $originalid skipped";
- printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
+ printlog(
+ {
+ id => $matched_record_id, op => "update",
+ status => "warning : already in database and option -update not enabled, skipping..."
+ }
+ ) if ($logfile);
}
- }
- elsif ($insert) {
+ } elsif ($insert) {
my $record_clone = $record->clone();
C4::Biblio::_strip_item_fields($record_clone);
- eval { ($record_id, $biblioitemnumber) = AddBiblio($record_clone, $framework, $add_biblio_options) };
+ eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) };
if ($@) {
warn "ERROR: Insert biblio $originalid failed: $@\n";
printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
next RECORD;
- }
- else {
+ } else {
printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile);
}
+
# If incoming record has bib ids set we need to transfer
# new ids from record_clone to incoming record to avoid
# working on wrong record (the original record) later on
# when adding items for example
- C4::Biblio::_koha_marc_update_bib_ids($record, $framework, $record_id, $biblioitemnumber);
- }
- else {
+ C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber );
+ } else {
warn "WARNING: Insert biblio $originalid skipped";
- printlog( { id => $originalid, op => "insert", status => "warning : biblio not in database and option -insert not enabled, skipping..." } ) if ($logfile);
+ printlog(
+ {
+ id => $originalid, op => "insert",
+ status => "warning : biblio not in database and option -insert not enabled, skipping..."
+ }
+ ) if ($logfile);
next RECORD;
}
my $record_has_added_items = 0;
if ($record_id) {
$yamlhash->{$originalid} = $record_id if $yamlfile;
- eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
+ eval {
+ ( $itemnumbers_ref, $errors_ref ) =
+ AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework );
+ };
my $error_adding = $@;
$record_has_added_items = @{$itemnumbers_ref};
if ($error_adding) {
warn "ERROR: Adding items to bib $record_id failed: $error_adding";
- printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
+ printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
+
# if we failed because of an exception, assume that
# the MARC columns in biblioitems were not set.
next RECORD;
}
- if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
+ if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' }
+ @$errors_ref )
+ {
# Find the record called 'barcode'
- my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
+ my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode');
+
# Now remove any items that didn't have a duplicate_barcode error,
# erase the barcodes on items that did, and re-add those items.
my %dupes;
- foreach my $i (0 .. $#{$errors_ref}) {
+ foreach my $i ( 0 .. $#{$errors_ref} ) {
my $ref = $errors_ref->[$i];
- if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
- $dupes{$ref->{item_sequence}} = 1;
+ if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) {
+ $dupes{ $ref->{item_sequence} } = 1;
+
# Delete the error message because we're going to
# retry this one.
delete $errors_ref->[$i];
}
}
my $seq = 0;
- foreach my $field ($record->field($tag)) {
+ foreach my $field ( $record->field($tag) ) {
$seq++;
- if ($dupes{$seq}) {
+ if ( $dupes{$seq} ) {
+
# Here we remove the barcode
- $field->delete_subfield(code => $sub);
- }
- else {
+ $field->delete_subfield( code => $sub );
+ } else {
+
# otherwise we delete the field because we don't want
# two of them
$record->delete_fields($field);
}
}
+
# Now re-add the record as before, adding errors to the prev list
my $more_errors;
- eval { ($itemnumbers_ref, $more_errors) =
- AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
+ eval {
+ ( $itemnumbers_ref, $more_errors ) =
+ AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' );
+ };
if ($@) {
warn "ERROR: Adding items to bib $record_id failed: $@\n";
- printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
+ printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
+
# if we failed because of an exception, assume that
# the MARC columns in biblioitems were not set.
next RECORD;
}
$record_has_added_items ||= @{$itemnumbers_ref};
- if (@{$more_errors}) {
+ if ( @{$more_errors} ) {
push @$errors_ref, @{$more_errors};
}
}
- if ($record_has_added_items) {
- printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
+ if ($record_has_added_items) {
+ printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile);
}
- if (@{$errors_ref}) {
- report_item_errors($record_id, $errors_ref);
+ if ( @{$errors_ref} ) {
+ report_item_errors( $record_id, $errors_ref );
}
my $biblio = Koha::Biblios->find($record_id);
$record = $biblio->metadata->record( { embed_items => 1 } );
push @search_engine_record_ids, $record_id;
- push @search_engine_records, $record;
+ push @search_engine_records, $record;
}
}
- if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) {
+ if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) {
$schema->txn_commit;
$schema->txn_begin;
if ($indexer) {
- $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
- if (C4::Context->preference('AutoLinkBiblios')) {
+ $indexer->update_index( \@search_engine_record_ids, \@search_engine_records );
+ if ( C4::Context->preference('AutoLinkBiblios') ) {
foreach my $record (@search_engine_records) {
- BiblioAutoLink($record, $framework);
+ BiblioAutoLink( $record, $framework );
}
}
@search_engine_record_ids = ();
- @search_engine_records = ();
+ @search_engine_records = ();
}
}
}
- print $record->as_formatted() . "\n" if ($verbose//0) == 2;
- last if $record_number == $number;
+ print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2;
+ last if $record_number == $number;
}
$schema->txn_commit;
@@ -679,104 +713,112 @@ if ($logfile) {
}
if ($yamlfile) {
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
- print $yamlfileout Encode::decode_utf8(YAML::XS::Dump($yamlhash));
+ print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) );
}
exit 0;
sub YAMLFileEntry {
- my ($record, $record_id, $updated) = @_;
+ my ( $record, $record_id, $updated ) = @_;
- my $entry = {
- authid => $record_id
- };
+ my $entry = { authid => $record_id };
# we recover all subfields of the heading authorities
my @subfields;
- foreach my $field ($record->field("2..")) {
+ foreach my $field ( $record->field("2..") ) {
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
}
$entry->{'subfields'} = \@subfields;
- $entry->{'updated'} = $updated;
+ $entry->{'updated'} = $updated;
return $entry;
}
sub RecordRevisionIsGtOrEq {
- my ($record_a, $record_b) = @_;
- return $record_a->field('005') && $record_b->field('005') &&
- $record_a->field('005')->data && $record_b->field('005')->data &&
- $record_a->field('005')->data >= $record_b->field('005')->data;
+ my ( $record_a, $record_b ) = @_;
+ return
+ $record_a->field('005')
+ && $record_b->field('005')
+ && $record_a->field('005')->data
+ && $record_b->field('005')->data
+ && $record_a->field('005')->data >= $record_b->field('005')->data;
}
sub GetRecordId {
my $marcrecord = shift;
- my $tag = shift;
- my $subfield = shift;
- if ($tag lt "010") {
+ my $tag = shift;
+ my $subfield = shift;
+ if ( $tag lt "010" ) {
return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
- }
- elsif ($subfield) {
- if ($marcrecord->field($tag)) {
- return $marcrecord->subfield($tag, $subfield);
+ } elsif ($subfield) {
+ if ( $marcrecord->field($tag) ) {
+ return $marcrecord->subfield( $tag, $subfield );
}
}
}
+
sub build_query {
- my ($match, $record) = @_;
+ my ( $match, $record ) = @_;
my @searchstrings;
foreach my $matchpoint (@$match) {
- my $query = build_simplequery($matchpoint, $record);
- push (@searchstrings, $query) if $query;
+ my $query = build_simplequery( $matchpoint, $record );
+ push( @searchstrings, $query ) if $query;
}
my $op = 'AND';
- return join(" $op ", @searchstrings);
+ return join( " $op ", @searchstrings );
}
+
sub build_simplequery {
- my ($matchpoint, $record) = @_;
+ my ( $matchpoint, $record ) = @_;
my @searchstrings;
- my ($index, $record_data) = split (/,/, $matchpoint);
- if ($record_data =~ /(\d{3})(.*)/) {
- my ($tag, $subfields) = ($1, $2);
- foreach my $field ($record->field($tag)) {
- if (length($field->as_string("$subfields")) > 0) {
- push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
+ my ( $index, $record_data ) = split( /,/, $matchpoint );
+ if ( $record_data =~ /(\d{3})(.*)/ ) {
+ my ( $tag, $subfields ) = ( $1, $2 );
+ foreach my $field ( $record->field($tag) ) {
+ if ( length( $field->as_string("$subfields") ) > 0 ) {
+ push( @searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"" );
}
}
- }
- else {
+ } else {
print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
}
my $op = 'AND';
- return join(" $op ", @searchstrings);
+ return join( " $op ", @searchstrings );
}
+
sub report_item_errors {
my $biblionumber = shift;
- my $errors_ref = shift;
+ my $errors_ref = shift;
- foreach my $error (@{ $errors_ref }) {
+ foreach my $error ( @{$errors_ref} ) {
next if !$error;
- my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
+ my $msg =
+ "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
my $error_code = $error->{'error_code'};
$error_code =~ s/_/ /g;
$msg .= "$error_code $error->{'error_information'}";
print $msg, "\n";
}
}
+
sub printlog {
my $logelements = shift;
- print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw}), "\n";
+ print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw} ), "\n";
}
+
sub get_heading_fields {
my $headingfields;
if ($authtypes) {
$headingfields = YAML::XS::LoadFile($authtypes);
$headingfields = { C4::Context->preference('marcflavour') => $headingfields };
- $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
+ $logger->debug( Encode::decode_utf8( YAML::XS::Dump($headingfields) ) );
}
unless ($headingfields) {
- $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
+ $headingfields = $dbh->selectall_hashref(
+ "SELECT auth_tag_to_report, authtypecode from auth_types", 'auth_tag_to_report',
+ { Slice => {} }
+ );
$headingfields = { C4::Context->preference('marcflavour') => $headingfields };
}
return $headingfields;
--