Attachment #127485 for bug #29440

View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

Lines 344-349 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here

(-)a/C4/Biblio.pm (-5 / +28 lines)
344	to authorities based on settings in the system preferences. This flag allows	344	to authorities based on settings in the system preferences. This flag allows
345	us to not relink records when the authority linker is saving modifications.	345	us to not relink records when the authority linker is saving modifications.
346		346
		347	=item C<defer_search_engine_indexing>
		348
		349	Don't update search index. Useful for bulk updates where this is handled
		350	manually for optimization purposes.
		351
347	=back	352	=back
348		353
349	Returns 1 on success 0 on failure	354	Returns 1 on success 0 on failure
Lines 353-358 Returns 1 on success 0 on failure Link Here
353	sub ModBiblio {	358	sub ModBiblio {
354	my ( $record, $biblionumber, $frameworkcode, $options ) = @_;	359	my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
355	$options //= {};	360	$options //= {};
		361	my %mod_biblio_marc_options;
		362	$mod_biblio_marc_options{'defer_search_engine_indexing'} =
		363	exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356		364
357	if (!$record) {	365	if (!$record) {
358	carp 'No record passed to ModBiblio';	366	carp 'No record passed to ModBiblio';
Lines 416-422 sub ModBiblio { Link Here
416	_koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );	424	_koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
417		425
418	# update the MARC record (that now contains biblio and items) with the new record data	426	# update the MARC record (that now contains biblio and items) with the new record data
419	ModBiblioMarc( $record, $biblionumber );	427	ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
420		428
421	# modify the other koha tables	429	# modify the other koha tables
422	_koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );	430	_koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2949-2966 sub _koha_delete_biblio_metadata { Link Here
2949		2957
2950	=head2 ModBiblioMarc	2958	=head2 ModBiblioMarc
2951		2959
2952	ModBiblioMarc($newrec,$biblionumber);	2960	ModBiblioMarc($newrec, $biblionumber, $options);
2953		2961
2954	Add MARC XML data for a biblio to koha	2962	Add MARC XML data for a biblio to koha
2955		2963
2956	Function exported, but should NOT be used, unless you really know what you're doing	2964	Function exported, but should NOT be used, unless you really know what you're doing
2957		2965
		2966	The C<$options> argument is a hashref with additional parameters:
		2967
		2968	=over 4
		2969
		2970	=item C<defer_search_engine_indexing>
		2971
		2972	Don't update search index. Useful for bulk updates where this is handled
		2973	manually for optimization purposes.
		2974
		2975	=back
		2976
2958	=cut	2977	=cut
2959		2978
2960	sub ModBiblioMarc {	2979	sub ModBiblioMarc {
2961	# pass the MARC::Record to this function, and it will create the records in	2980	# pass the MARC::Record to this function, and it will create the records in
2962	# the marcxml field	2981	# the marcxml field
2963	my ( $record, $biblionumber ) = @_;	2982	my ( $record, $biblionumber, $options ) = @_;
		2983	$options //= {};
		2984
2964	if ( !$record ) {	2985	if ( !$record ) {
2965	carp 'ModBiblioMarc passed an undefined record';	2986	carp 'ModBiblioMarc passed an undefined record';
2966	return;	2987	return;
Lines 3025-3032 sub ModBiblioMarc { Link Here
3025	$m_rs->metadata( $record->as_xml_record($encoding) );	3046	$m_rs->metadata( $record->as_xml_record($encoding) );
3026	$m_rs->store;	3047	$m_rs->store;
3027		3048
3028	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });	3049	unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
3029	$indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );	3050	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		3051	$indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
		3052	}
3030		3053
3031	return $biblionumber;	3054	return $biblionumber;
3032	}	3055	}




    GetMarcFromKohaField
    ModBiblio
    ModBiblioMarc
    GetFrameworkCode
    GetMarcBiblio
);
use C4::Koha;
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );

use Getopt::Long qw( GetOptions );
use IO::File;
use Pod::Usage qw( pod2usage );
use List::MoreUtils qw(any);

use Koha::Logger;
use Koha::Biblios;
use Koha::SearchEngine;
use Koha::SearchEngine::Search;
use Koha::Plugins::Handler;

use open qw( :std :encoding(UTF-8) );
binmode(STDOUT, ":encoding(UTF-8)");
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
my $version;
my $delete;
my $test_parameter;
my $skip_marc8_conversion;
my $char_encoding;
my $verbose;
my $commit;
my $fk_off;
my $format;
my $biblios;
my $authorities;
my $keepids;
my $match;
my $isbn_check;
my $logfile;
my $insert;
my $filters;
my $update;
my $all;
my $yamlfile;
my $authtypes;
my $append;
my $sourcetag;
my $sourcesubfield;
my $idmapfl;
my $dedup_barcode;
my $framework = '';
my $localcust;
my $marc_mod_template = '';
my $marc_mod_template_id = -1;
$| = 1;
$|=1;

GetOptions(
    'commit:f' => \$commit,
    'file:s' => \$input_marc_file,
    'n:f' => \$number,
    'o|offset:f' => \$offset,
    'h' => \$version,

    'b|biblios' => \$biblios,
    'a|authorities' => \$authorities,
    'authtypes:s' => \$authtypes,
    'filter=s@' => \$filters,
    'insert' => \$insert,
    'update' => \$update,
    'all' => \$all,
    'match=s@' => \$match,
    'i|isbn' => \$isbn_check,
    'x:s' => \$sourcetag,
    'y:s' => \$sourcesubfield,
    'idmap:s' => \$idmapfl,
    'cleanisbn!' => \$cleanisbn,
    'yaml:s' => \$yamlfile,
    'dedupbarcode' => \$dedup_barcode,
    'framework=s' => \$framework,
    'custom:s' => \$localcust,
    'marcmodtemplate:s' => \$marc_mod_template,
);

$biblios ||= !$authorities;
$insert  ||= !$update;
my $writemode = ($append) ? "a" : "w";

    $update = 1;
}

my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
my $modify_biblio_marc_options = {
    defer_search_engine_indexing => $using_elastic_search,
    overlay_context => { source => 'bulkmarcimport' }
};

my @search_engine_record_ids;
my @search_engine_records;
my $indexer;
if ($using_elastic_search) {
    use Koha::SearchEngine::Elasticsearch::Indexer;
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
        { index => $authorities ?
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
        }
    );
}

if ($version || ($input_marc_file eq '')) {
    pod2usage( -verbose => 2 );
    exit;

    warn "Using -update without -match or -isbn seems to be useless.\n";
}

if (defined $localcust) { #local customize module
    if (!-e $localcust) {
        $localcust = $localcust || 'LocalChanges'; #default name
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
        $localcust =~ s/\.pm$//;           #remove extension
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
        if (-e $fqcust) {
            $localcust= $fqcust;
        }
        else {

        }
    }
    require $localcust if $localcust;
    $localcust = \&customize if $localcust;
}

if($marc_mod_template ne '') {

}

my $dbh = C4::Context->dbh;
my $heading_fields = get_heading_fields();

my $idmapfh;

if (defined $idmapfl) {
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
}

if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
    $sourcetag = "910";
    $sourcesubfield = "a";
}


# Disable logging for the biblios and authorities import operation. It would unnecessarily
# slow the import
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;

if ($fk_off) {
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
}


if ($delete) {
    if ($biblios) {
        print "Deleting biblios\n";
        $dbh->do("DELETE FROM biblio");
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
        $dbh->do("DELETE FROM biblioitems");
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
        $dbh->do("DELETE FROM items");
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
    }
    else {
        print "Deleting authorities\n";
        $dbh->do("truncate auth_header");
    }
    $dbh->do("truncate zebraqueue");
}



if ($test_parameter) {
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
}

my $batch;
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';

# The definition of $searcher must be before MARC::Batch->new
my $searcher = Koha::SearchEngine::Search->new(
    {
        index => (
            $authorities
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
            : $Koha::SearchEngine::BIBLIOS_INDEX
        )
    }
);

print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
my $starttime = gettimeofday;

my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
if (defined $format && $format =~ /XML/i) {
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,

    #       extract the records, not using regexes to look
    #       for <record>.*</record>.
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
    $batch = MARC::Batch->new('XML', $fh);
}
else {
    $batch = MARC::Batch->new('USMARC', $fh);
}

$batch->warnings_off();
$batch->strict_off();
my $i=0;
my $commitnum = $commit ? $commit : 50;
my $yamlhash;

# Skip file offset
if ($offset) {
    print "Skipping file offset: $offset records\n";
    $batch->next() while ($offset--);
}

my ($tagid, $subfieldid);
if ($authorities) {
    $tagid = '001';
}
else {
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
    $tagid ||= "001";
	$tagid||="001";
}

my $sth_isbn;
# the SQL query to search on isbn
if ($isbn_check) {
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
}

my $loghandle;
if ($logfile) {
    $loghandle= IO::File->new($logfile, $writemode);
    print $loghandle "id;operation;status\n";
}

my $record_number = 0;
my $logger = Koha::Logger->get;
my $schema = Koha::Database->schema;
$schema->txn_begin;
my $marc_records = [];
RECORD: while (  ) {
    my $record;
    $record_number++;
    # get record
    eval { $record = $batch->next() };
    if ($@) {
        print "Bad MARC record $record_number: $@ skipped\n";
        # FIXME - because MARC::Batch->next() combines grabbing the next
        # blob and parsing it into one operation, a correctable condition
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered

        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
        next;
    }
    if ($record) {
        # transcode the record to UTF8 if needed & applicable.
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
            my ($guessed_charset, $charset_errors);
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
            if ($guessed_charset eq 'failed') {
                warn "ERROR: failed to perform character conversion for record $record_number\n";
                next RECORD;
            }
        }
        SetUTF8Flag($record);
        &$localcust($record) if $localcust;
        push @{$marc_records}, $record;
    }
    else {
        last;
    }
}

$record_number = 0;
my $records_total = @{$marc_records};
$schema->txn_begin;
RECORD: foreach my $record (@{$marc_records}) {
    $record_number++;
    if (($verbose//1) == 1) { #no dot for verbose==2
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
            next RECORD;            
        }
    }

    if ($marc_mod_template_id > 0) {
        print "Modifying MARC\n" if $verbose;
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
    }

    my $isbn;
    # remove trailing - in isbn (only for biblios, of course)
    if ($biblios) {
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
        my $field = $record->field($tag);
        my $isbn = $field && $field->subfield('a');
        if ( $isbn && $cleanisbn ) {
            $isbn =~ s/-//g;
            $field->update('a' => $isbn);
        }
    }
    my $id;
    # search for duplicates (based on Local-number)
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
    my $matched_record_id = undef;
    if ($match) {
        require C4::Search;
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
        my $query = build_query($match, $record);
        $logger->debug("Bulkmarcimport: $query");
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
        # changed to warn so able to continue with one broken record
        if (defined $error) {
            warn "unable to search the database for duplicates : $error";
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
            next RECORD;
        }
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
        # sub SimpleSearch could return undefined, but only on error, so
        # should not really need to safeguard here, but do so anyway
        $results //= [];
        if (@{$results} == 1) {
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
            SetUTF8Flag($matched_record);
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);

            if ($authorities && $marc_flavour) {
                #Skip if authority in database is the same or newer than the incoming record
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
                    if ($yamlfile) {
                        $yamlhash->{$originalid} = YAMLFileEntry(
                            $matched_record,
                            $matched_record_id,
                            0
                        );
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
                        }
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
                        $yamlhash->{$originalid}->{'updated'} = 0;
                    }
                    next;
                }
            }
        } elsif ( $results && scalar(@$results) > 1 ) {
            $logger->debug("more than one match for $query");
        } else {
            $logger->debug("nomatch for $query");
        }
        elsif(@{$results} > 1) {
            $logger->debug("More than one match for: $query");
        }
        else {
            $logger->debug("No match for: $query");
        }

        if ($keepids && $originalid) {
            my $storeidfield;
            if (length($keepids) == 3) {
                $storeidfield = MARC::Field->new($keepids, $originalid);
            } else {
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
            }
            $record->insert_fields_ordered($storeidfield);
            $record->delete_field($record->field($tagid));
        }
    }

    foreach my $stringfilter (@$filters) {
        if (length($stringfilter) == 3) {
            foreach my $field ($record->field($stringfilter)) {
                $record->delete_field($field);
                $logger->debug("Removed: ", $field->as_string);
            }
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
            my $removetag = $1;
            my $removesubfield = $2;
            my $removematch = $3;
            if (($removetag > "010") && $removesubfield) {
                foreach my $field ($record->field($removetag)) {
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
                }
            }
        }
    }
    unless ($test_parameter) {
        if ($authorities) {
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
            if ($authid && GetAuthority($authid) && $update) {
            ## Authority has an id and is in database : Replace
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
                if ($@) {
                    warn "Problem with authority $authid Cannot Modify";
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
                }
                else{
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
                }
            }
            else {
				}
            }  
	        else {
            ## True insert in database
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
                if ($@) {
                    warn "Problem with authority $originalid Cannot Add ".$@;
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
                }
                else {
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
                }

            if ($yamlfile) {
            $yamlhash->{$originalid}->{'authid'} = $authid;
            my @subfields;
            foreach my $field ( $record->field("2..") ) {
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
            }
            if ($yamlfile) {
                $yamlhash->{$originalid} = YAMLFileEntry(
                    $record,
                    $authid,
                    1 #@FIXME: Really always updated?
                );
            }
        }
        else {
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
            $biblionumber = $id;
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
            if (!$matched_record_id && $isbn_check) {
                my $field = $record->field($marc_flavour eq 'UNIMARC' ? '010' : '020');
                my $isbn = $field && $field->subfield('a');
                # TODO: Check that isbn has valid valiue, or $field->subfield('a')->data??
                $sth_isbn->execute($isbn);
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
            }

            if (defined $idmapfl && $matched_record_id) {
                if ($sourcetag < "010") {
                    if ($record->field($sourcetag)) {
                        my $source = $record->field($sourcetag)->data();
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
                    }
                }
                else {
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
                }
            }

            # create biblio, unless we already have it ( either match or isbn )
            if ($matched_record_id) {
                # TODO: Implement also for authority records!
                eval{
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
                };
                if ($update) {
                    my $success;
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
                    if ($@) {
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
                        next RECORD;
                    } else {
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
                    }
                    elsif (!$success) {
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
            } else {
                if ($insert) {
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
                    if ($@) {
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
                        next RECORD;
                    } else {
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
                    }
                    else {
                        $record_id = $matched_record_id;
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
                    }
                }
                else {
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
                }
            }
            elsif ($insert) {
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
                if ($@) {
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
                    next RECORD;
                }
                else {
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
                }
            }
            else {
                warn "WARNING: Updating record ".($originalid)." failed";
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
            # fix them up later.
			my $clone_record = $record->clone();
            C4::Biblio::_strip_item_fields($clone_record, '');
            # This sets the marc fields if there was an error, and also calls
            # defer_marc_save.
            ModBiblioMarc( $clone_record, $biblionumber );
            if ( $error_adding ) {
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
                # if we failed because of an exception, assume that 
                # the MARC columns in biblioitems were not set.
                next RECORD;
            }
            my $record_has_added_items = 0;
            if ($record_id) {
                $yamlhash->{$originalid} = $record_id if $yamlfile;
                # TODO: Add option for skipping items?
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
                $record_has_added_items = @{$itemnumbers_ref};
                my $error_adding = $@;
                # Work on a clone so that if there are real errors, we can maybe
                # fix them up later.
                my $clone_record = $record->clone();
                C4::Biblio::_strip_item_fields($clone_record, $framework);
                # This sets the marc fields if there was an error, and also calls
                # defer_marc_save.
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
                if ($error_adding) {
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
                }
                my $seq = 0;
                foreach my $field ($record->field($tag)) {
                    $seq++;
                    if ($dupes{$seq}) {
                        # Here we remove the barcode
                        $field->delete_subfield(code => $sub);
                    } else {
                        # otherwise we delete the field because we don't want
                        # two of them
                        $record->delete_fields($field);
                    }
                }
                # Now re-add the record as before, adding errors to the prev list
                my $more_errors;
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
                if ( $@ ) {
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
                    # if we failed because of an exception, assume that
                    # the MARC columns in biblioitems were not set.
                    ModBiblioMarc( $record, $biblionumber );
                    next RECORD;
                } else {
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
                }
                else {
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
                }
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
                    # Find the record called 'barcode'
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
                    # Now remove any items that didn't have a duplicate_barcode error,
                    # erase the barcodes on items that did, and re-add those items.
                    my %dupes;
                    # FIXME: This could cause array out of bounds because shifting down rest of items on array item delete?
                    foreach my $i (0 .. $#{$errors_ref}) {
                        my $ref = $errors_ref->[$i];
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
                            $dupes{$ref->{item_sequence}} = 1;
                            # Delete the error message because we're going to
                            # retry this one.
                            delete $errors_ref->[$i];
                        }
                    }
                    my $seq = 0;
                    foreach my $field ($record->field($tag)) {
                        $seq++;
                        if ($dupes{$seq}) {
                            # Here we remove the barcode
                            $field->delete_subfield(code => $sub);
                        }
                        else {
                            # otherwise we delete the field because we don't want
                            # two of them
                            $record->delete_fields($field);
                        }
                    }
                    # Now re-add the record as before, adding errors to the prev list
                    my $more_errors;
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
                    $record_has_added_items ||= @{$itemnumbers_ref};
                    if ($@) {
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
                        # if we failed because of an exception, assume that
                        # the MARC columns in biblioitems were not set.

                        # @FIXME: Why do we save here without stripping items? Besides,
                        # save with stripped items has already been performed
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
                        next RECORD;
                    }
                    else {
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
                    }
                    push @$errors_ref, @{$more_errors};
                }
                if (@{$errors_ref}) {
                    report_item_errors($record_id, $errors_ref);
                }
                C4::Biblio::_strip_item_fields($record, $framework);
                if ($record_has_added_items || $matched_record_id) {
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
                }
                push @search_engine_record_ids, $record_id;
                push @search_engine_records, $record;
            }
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
        }
        if ($record_number % $commitnum == 0 || $record_number == $records_total) {
            $schema->txn_commit;
            $schema->txn_begin;
            if ($indexer) {
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
                @search_engine_record_ids = ();
                @search_engine_records = ();
            }
        }
    }
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
    last if $record_number == $number;
}
$schema->txn_commit;

if ($fk_off) {
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
}

# Restore CataloguingLog and AuthoritiesLog

delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};

my $timeneeded = gettimeofday - $starttime;
print "\n$record_number MARC records done in $timeneeded seconds\n";
if ($logfile) {
    print $loghandle "file : $input_marc_file\n";
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
    $loghandle->close;
}
if ($yamlfile) {
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";

}
exit 0;

sub YAMLFileEntry {
    my ($record, $record_id, $updated) = @_;

    my $entry = {
        authid => $record_id
    };

    # we recover all subfields of the heading authorities
    my @subfields;
    foreach my $field ($record->field("2..")) {
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
    }
    $entry->{'subfields'} = \@subfields;
    $entry->{'updated'} = $updated;

    return $entry;
}

sub RecordRevisionIsGtOrEq {
    my ($record_a, $record_b) = @_;
    return $record_a->field('005') && $record_b->field('005') &&
    $record_a->field('005')->data && $record_b->field('005')->data &&
    $record_a->field('005')->data >= $record_b->field('005')->data;
}

sub GetRecordId {
    my $marcrecord = shift;
    my $tag = shift;
    my $subfield = shift;
    if ($tag lt "010") {
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
    }
    elsif ($subfield) {
        if ($marcrecord->field($tag)) {
            return $marcrecord->subfield($tag, $subfield);
        }
    }
    return undef;
}
sub build_query {
    my ($match, $record) = @_;
    my @searchstrings;

    foreach my $matchpoint (@$match) {
        my $query = build_simplequery($matchpoint, $record);
        push (@searchstrings, $query) if $query;
    }
    my $op = 'and';
    return join(" $op ", @searchstrings);
}
sub build_simplequery {
    my ($matchpoint, $record) = @_;

    my @searchstrings;
    my ($index, $record_data) = split (/,/, $matchpoint);
    if ($record_data =~ /(\d{3})(.*)/) {
        my ($tag, $subfields) = ($1, $2);
        foreach my $field ($record->field($tag)) {
            if (length($field->as_string("$subfields")) > 0) {
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
            }
        }
    }
    else {
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
    }
    my $op = 'and';
    return join(" $op ", @searchstrings);
}
sub report_item_errors {
    my $biblionumber = shift;

        print $msg, "\n";
    }
}
sub printlog {
    my $logelements = shift;
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
}
sub get_heading_fields {
    my $headingfields;
    if ($authtypes) {
        $headingfields = YAML::XS::LoadFile($authtypes);
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
    }
    unless ($headingfields) {
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
    }
    return $headingfields;
}
- 

Return to bug 29440