=item B<skip_record_index>: used when the indexing schedulling will be handled by the caller

=item B<skip_record_index>: used when the indexing scheduling will be handled by the caller

    my $skip_record_index = $options->{skip_record_index} || 0;

    my $skip_record_index = $options->{'skip_record_index'} // 0;

    my $defer_marc_save   = $options->{defer_marc_save}   || 0;

    my $defer_marc_save = $options->{defer_marc_save} // 0;

    my $skip_record_index = $options->{skip_record_index} || 0;

    my $mod_biblio_marc_options = {

        skip_record_index => $options->{'skip_record_index'} // 0

    };

    ModBiblioMarc( $record, $biblionumber, { skip_record_index => $skip_record_index } );

    ModBiblioMarc( $record, $biblionumber, $mod_biblio_marc_options );

  ModBiblioMarc($newrec,$biblionumber);

  ModBiblioMarc($newrec, $biblionumber, $options);

    my ( $record, $biblionumber, $params ) = @_;

    my ( $record, $biblionumber, $options ) = @_;

    $options //= {};

    my $skip_record_index = $params->{skip_record_index} || 0;

    my $skip_record_index = $options->{skip_record_index} // 0;

binmode( STDOUT, ":encoding(UTF-8)" );

binmode(STDOUT, ":encoding(UTF-8)");

my ( $input_marc_file, $number, $offset) = ('',0,0);

my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);

my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);

my $version;

my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );

my $delete;

my $cleanisbn = 1;

my $test_parameter;

my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);

my $skip_marc8_conversion;

my $char_encoding;

my $verbose;

my $commit;

my $fk_off;

my $format;

my $biblios;

my $authorities;

my $keepids;

my $match;

my $isbn_check;

my $logfile;

my $insert;

my $filters;

my $update;

my $all;

my $yamlfile;

my $authtypes;

my $append;

my $sourcetag;

my $sourcesubfield;

my $idmapfl;

my $dedup_barcode;

$| = 1;

$|=1;

    'commit:f'    => \$commit,

    'commit:f' => \$commit,

    'file:s'    => \$input_marc_file,

    'file:s' => \$input_marc_file,

    'filter=s@'     => \$filters,

    'filter=s@' => \$filters,

    'insert'        => \$insert,

    'insert' => \$insert,

    'update'        => \$update,

    'update' => \$update,

    'all'           => \$all,

    'all' => \$all,

    'match=s@'    => \$match,

    'match=s@' => \$match,

    'cleanisbn!'     => \$cleanisbn,

    'cleanisbn!' => \$cleanisbn,

    'yaml:s'        => \$yamlfile,

    'yaml:s' => \$yamlfile,

    'custom:s'    => \$localcust,

    'custom:s' => \$localcust,

if(defined $localcust) { #local customize module

if (defined $localcust) { #local customize module

    if(!-e $localcust) {

    if (!-e $localcust) {

        $localcust= $localcust||'LocalChanges'; #default name

        $localcust = $localcust || 'LocalChanges'; #default name

        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only

        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only

        $localcust=~ s/\.pm$//;           #remove extension

        $localcust =~ s/\.pm$//;           #remove extension

        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir

        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir

        if(-e $fqcust) {

        if (-e $fqcust) {

    $localcust=\&customize if $localcust;

    $localcust = \&customize if $localcust;

my $heading_fields=get_heading_fields();

my $heading_fields = get_heading_fields();

if ((not defined $sourcesubfield) && (not defined $sourcetag)){

if ((not defined $sourcesubfield) && (not defined $sourcetag)) {

  $sourcetag="910";

    $sourcetag = "910";

  $sourcesubfield="a";

    $sourcesubfield = "a";

	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");

    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");

	if ($biblios){

    if ($biblios) {

    	print "deleting biblios\n";

        print "Deleting biblios\n";

	}

    }

	else {

    else {

    	print "deleting authorities\n";

        print "Deleting authorities\n";

    	$dbh->do("truncate auth_header");

        $dbh->do("truncate auth_header");

	}

    }

my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';

my $batch;

my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';

              $authorities

            $authorities

print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;

print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;

my $batch;

    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));

    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));

#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.

    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.

    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");

    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");

    $batch = MARC::Batch->new( 'XML', $fh );

    $batch = MARC::Batch->new('XML', $fh);

} else {

}

    $batch = MARC::Batch->new( 'USMARC', $fh );

else {

    $batch = MARC::Batch->new('USMARC', $fh);

if ( $offset ) {

if ($offset) {

my ($tagid,$subfieldid);

my ($tagid, $subfieldid);

if ($authorities){

if ($authorities) {

	  $tagid='001';

    $tagid = '001';

   ( $tagid, $subfieldid ) =

    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");

            GetMarcFromKohaField( "biblio.biblionumber" );

    $tagid ||= "001";

	$tagid||="001";

my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");

if ($isbn_check) {

    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");

}

if ($logfile){

if ($logfile) {

   $loghandle= IO::File->new($logfile, $writemode) ;

    $loghandle= IO::File->new($logfile, $writemode);

   print $loghandle "id;operation;status\n";

    print $loghandle "id;operation;status\n";

$schema->txn_begin;

my $marc_records = [];

    # get records

    $record_number++;

    # get record

    if ( $@ ) {

    if ($@) {

        print "Bad MARC record $i: $@ skipped\n";

        print "Bad MARC record $record_number: $@ skipped\n";

    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure

    if ($record) {

    last unless ( $record );

        # transcode the record to UTF8 if needed & applicable.

    $i++;

        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {

    if( ($verbose//1)==1 ) { #no dot for verbose==2

            my ($guessed_charset, $charset_errors);

        print "." . ( $i % 100==0 ? "\n$i" : '' );

            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));

            if ($guessed_charset eq 'failed') {

                warn "ERROR: failed to perform character conversion for record $record_number\n";

                next RECORD;

            }

        }

        SetUTF8Flag($record);

        &$localcust($record) if $localcust;

        push @{$marc_records}, $record;

    }

    else {

        last;

    # transcode the record to UTF8 if needed & applicable.

$record_number = 0;

    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {

my $records_total = @{$marc_records};

        # FIXME update condition

$schema->txn_begin;

        my ($guessed_charset, $charset_errors);

RECORD: foreach my $record (@{$marc_records}) {

         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));

    $record_number++;

        if ($guessed_charset eq 'failed') {

    if (($verbose//1) == 1) { #no dot for verbose==2

            warn "ERROR: failed to perform character conversion for record $i\n";

        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');

            next RECORD;            

        }

    SetUTF8Flag($record);

    if($marc_mod_template_id > 0) {

    if ($marc_mod_template_id > 0) {

    print "Modifying MARC\n" if $verbose;

        print "Modifying MARC\n" if $verbose;

    ModifyRecordWithTemplate( $marc_mod_template_id, $record );

        ModifyRecordWithTemplate( $marc_mod_template_id, $record );

    &$localcust($record) if $localcust;

    if( $biblios ) {

    if ($biblios && ($cleanisbn || $isbn_check)) {

        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';

        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';

        if ( $isbn && $cleanisbn ) {

        if ($isbn && $cleanisbn) {

    my $originalid;

    my $originalid = GetRecordId($record, $tagid, $subfieldid);

    $originalid = GetRecordId( $record, $tagid, $subfieldid );

    my $matched_record_id = undef;

        my $query = build_query( $match, $record );

        my $server = ($authorities ? 'authorityserver' : 'biblioserver');

        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );

        my $query = build_query($match, $record);

        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );

        $logger->debug("Bulkmarcimport: $query");

        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);

        if ( defined $error ) {

        if (defined $error) {

            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);

            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);

        if ( $results && scalar(@$results) == 1 ) {

        $logger->debug("Bulkmarcimport: $query $server : $totalhits");

            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );

        # sub SimpleSearch could return undefined, but only on error, so

            SetUTF8Flag($marcrecord);

        # should not really need to safeguard here, but do so anyway

            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );

        $results //= [];

            if ( $authorities && $marcFlavour ) {

        if (@{$results} == 1) {

                #Skip if authority in database is the same as the on in database

            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);

                if ( $marcrecord->field('005') && $record->field('005') &&

            SetUTF8Flag($matched_record);

                     $marcrecord->field('005')->data && $record->field('005')->data &&

            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);

                     $marcrecord->field('005')->data >= $record->field('005')->data ) {

            if ($authorities && $marc_flavour) {

                #Skip if authority in database is the same or newer than the incoming record

                if (RecordRevisionIsGtOrEq($matched_record, $record)) {

                        $yamlhash->{$originalid}->{'authid'} = $id;

                        $yamlhash->{$originalid} = YAMLFileEntry(

                            $matched_record,

                        # we recover all subfields of the heading authorities

                            $matched_record_id,

                        my @subfields;

                            0

                        foreach my $field ( $marcrecord->field("2..") ) {

                        );

                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();

                        }

                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;

                        $yamlhash->{$originalid}->{'updated'} = 0;

    }

        elsif(@{$results} > 1) {

    if ($keepids && $originalid) {

            $logger->debug("More than one match for: $query");

        }

        else {

            $logger->debug("No match for: $query");

        }

        if ($keepids && $originalid) {

            if ( length($keepids) == 3 ) {

            if (length($keepids) == 3) {

                $storeidfield = MARC::Field->new( $keepids, $originalid );

                $storeidfield = MARC::Field->new($keepids, $originalid);

                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );

                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);

            $record->delete_field( $record->field($tagid) );

            $record->delete_field($record->field($tagid));

        }

        if ( length($stringfilter) == 3 ) {

        if (length($stringfilter) == 3) {

            foreach my $field ( $record->field($stringfilter) ) {

            foreach my $field ($record->field($stringfilter)) {

                $logger->debug("removed : ", $field->as_string);

                $logger->debug("Removed: ", $field->as_string);

            if ( ( $removetag > "010" ) && $removesubfield ) {

            if (($removetag > "010") && $removesubfield) {

                foreach my $field ( $record->field($removetag) ) {

                foreach my $field ($record->field($removetag)) {

                    $field->delete_subfield( code => "$removesubfield", match => $removematch );

                    $field->delete_subfield(code => "$removesubfield", match => $removematch);

                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));

                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));

        if ($authorities){

        if ($authorities) {

            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);

            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);

            my $authid= ($id?$id:GuessAuthId($record));

            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));

            if ($authid && GetAuthority($authid) && $update ){

            if ($authid && GetAuthority($authid) && $update) {

                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };

                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };

                if ($@){

                if ($@) {

					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);

                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);

				else{

                else{

					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);

                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);

				}

                }

            }  

            }

	        else {

            else {

                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };

                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };

                if ($@){

                if ($@) {

                    warn "Problem with authority $authid Cannot Add".$@;

                    warn "Problem with authority $originalid Cannot Add ".$@;

					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);

                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);

   				else{

                else {

					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);

                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);

				}

                }

 	        }

            if ($yamlfile) {

            $yamlhash->{$originalid}->{'authid'} = $authid;

            my @subfields;

            foreach my $field ( $record->field("2..") ) {

                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();

            $yamlhash->{$originalid}->{'subfields'} = \@subfields;

            if ($yamlfile) {

            $yamlhash->{$originalid}->{'updated'} = 1;

                $yamlhash->{$originalid} = YAMLFileEntry(

                    $record,

                    $authid,

                    1 #@FIXME: Really always updated?

                );

            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );

            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);

            $biblionumber = $id;

            if (!$biblionumber && $isbn_check && $isbn) {

            if (!$matched_record_id && $isbn_check && $isbn) {

    #         warn "search ISBN : $isbn";

                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;

                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;

            }

            if (defined $idmapfl && $matched_record_id) {

                if ($sourcetag < "010") {

                    if ($record->field($sourcetag)) {

                        my $source = $record->field($sourcetag)->data();

                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);

                    }

                }

                else {

                    my $source = $record->subfield($sourcetag, $sourcesubfield);

                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);

                }

        	if (defined $idmapfl) {

			 	if ($sourcetag < "010"){

            # create biblio, unless we already have it ( either match or isbn )

					if ($record->field($sourcetag)){

            if ($matched_record_id) {

					  my $source = $record->field($sourcetag)->data();

                # TODO: Implement also for authority records!

                      printf($idmapfh "%s|%s\n",$source,$biblionumber);

					}

			    } else {

					my $source=$record->subfield($sourcetag,$sourcesubfield);

                    printf($idmapfh "%s|%s\n",$source,$biblionumber);

			  }

			}

					# create biblio, unless we already have it ( either match or isbn )

            if ($biblionumber) {

                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;

                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;

                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };

                    my $success;

                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };

                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";

                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";

                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);

                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);

                } else {

                    elsif (!$success) {

                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);

                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";

                }

                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);

            } else {

                if ($insert) {

                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };

                    if ($@) {

                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";

                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);

                } else {

                    else {

                    warn "WARNING: Updating record ".($id||$originalid)." failed";

                        $record_id = $matched_record_id;

                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);

                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);

                    }

                }

                else {

                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);

                }

            }

            elsif ($insert) {

                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };

                if ($@) {

                    warn "ERROR: Adding biblio $record_id failed: $@\n";

                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);

            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };

            else {

            my $error_adding = $@;

                warn "WARNING: Updating record ".($originalid)." failed";

            # Work on a clone so that if there are real errors, we can maybe

                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);

            # fix them up later.

			my $clone_record = $record->clone();

            C4::Biblio::_strip_item_fields($clone_record, '');

            # This sets the marc fields if there was an error, and also calls

            # defer_marc_save.

            ModBiblioMarc( $clone_record, $biblionumber );

            if ( $error_adding ) {

                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";

				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);

                # if we failed because of an exception, assume that 

                # the MARC columns in biblioitems were not set.

 			else{

            my $record_has_added_items = 0;

				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);

            if ($record_id) {

			}

                $yamlhash->{$originalid} = $record_id if $yamlfile;

            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {

                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };

                # Find the record called 'barcode'

                $record_has_added_items = @{$itemnumbers_ref};

                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );

                my $error_adding = $@;

                # Now remove any items that didn't have a duplicate_barcode error,

                # Work on a clone so that if there are real errors, we can maybe

                # erase the barcodes on items that did, and re-add those items.

                # fix them up later.

                my %dupes;

                my $clone_record = $record->clone();

                foreach my $i (0 .. $#{$errors_ref}) {

                C4::Biblio::_strip_item_fields($clone_record, $framework);

                    my $ref = $errors_ref->[$i];

                # This sets the marc fields if there was an error, and also calls

                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {

                # defer_marc_save.

                        $dupes{$ref->{item_sequence}} = 1;

                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);

                        # Delete the error message because we're going to

                if ($error_adding) {

                        # retry this one.

                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";

                        delete $errors_ref->[$i];

                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);

                    }

                }

                my $seq = 0;

                foreach my $field ($record->field($tag)) {

                    $seq++;

                    if ($dupes{$seq}) {

                        # Here we remove the barcode

                        $field->delete_subfield(code => $sub);

                    } else {

                        # otherwise we delete the field because we don't want

                        # two of them

                        $record->delete_fields($field);

                    }

                }

                # Now re-add the record as before, adding errors to the prev list

                my $more_errors;

                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };

                if ( $@ ) {

                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";

                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);

                push @$errors_ref, @{ $more_errors };

                else {

            }

                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);

            if ($#{ $errors_ref } > -1) {

                }

                report_item_errors($biblionumber, $errors_ref);

                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {

                    # Find the record called 'barcode'

                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');

                    # Now remove any items that didn't have a duplicate_barcode error,

                    # erase the barcodes on items that did, and re-add those items.

                    my %dupes;

                    foreach my $i (0 .. $#{$errors_ref}) {

                        my $ref = $errors_ref->[$i];

                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {

                            $dupes{$ref->{item_sequence}} = 1;

                            # Delete the error message because we're going to

                            # retry this one.

                            delete $errors_ref->[$i];

                        }

                    }

                    my $seq = 0;

                    foreach my $field ($record->field($tag)) {

                        $seq++;

                        if ($dupes{$seq}) {

                            # Here we remove the barcode

                            $field->delete_subfield(code => $sub);

                        }

                        else {

                            # otherwise we delete the field because we don't want

                            # two of them

                            $record->delete_fields($field);

                        }

                    }

                    # Now re-add the record as before, adding errors to the prev list

                    my $more_errors;

                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };

                    $record_has_added_items ||= @{$itemnumbers_ref};

                    if ($@) {

                        warn "ERROR: Adding items to bib $record_id failed: $@\n";

                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);

                        # if we failed because of an exception, assume that

                        # the MARC columns in biblioitems were not set.

                        # @FIXME: Why do we save here without stripping items? Besides,

                        # save with stripped items has already been performed

                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);

                        next RECORD;

                    }

                    else {

                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);

                    }

                    push @$errors_ref, @{$more_errors};

                }

                if (@{$errors_ref}) {

                    report_item_errors($record_id, $errors_ref);

                }

                C4::Biblio::_strip_item_fields($record, $framework);

                if ($record_has_added_items || $matched_record_id) {

                    # Replace with record from GetMarcBiblio with "$embeditems = 1"

                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});

                }

                push @search_engine_record_ids, $record_id;

                push @search_engine_records, $record;

        if ( 0 == $i % $commitnum ) {

        if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) {

    print $record->as_formatted()."\n" if ($verbose//0)==2;

    print $record->as_formatted() . "\n" if ($verbose//0) == 2;

    last if $i == $number;

    last if $record_number == $number;

	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");

    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");

print "\n$i MARC records done in $timeneeded seconds\n";

print "\n$record_number MARC records done in $timeneeded seconds\n";

if ($logfile){

if ($logfile) {

  print $loghandle "file : $input_marc_file\n";

    print $loghandle "file : $input_marc_file\n";

  print $loghandle "$i MARC records done in $timeneeded seconds\n";

    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";

  $loghandle->close;

    $loghandle->close;

sub GetRecordId{

sub YAMLFileEntry {

	my $marcrecord=shift;

    my ($record, $record_id, $updated) = @_;

	my $tag=shift;

	my $subfield=shift;

    my $entry = {

	my $id;

        authid => $record_id

	if ($tag lt "010"){

    };

		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);

	} 

    # we recover all subfields of the heading authorities

	elsif ($subfield){

    my @subfields;

		if ($marcrecord->field($tag)){

    foreach my $field ($record->field("2..")) {

			return $marcrecord->subfield($tag,$subfield);

        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();

		}

    }

	}

    $entry->{'subfields'} = \@subfields;

	return $id;

    $entry->{'updated'} = $updated;

    return $entry;

sub build_query {

	my $match = shift;

sub RecordRevisionIsGtOrEq {

	my $record=shift;

    my ($record_a, $record_b) = @_;

        my @searchstrings;

    return $record_a->field('005') && $record_b->field('005') &&

	foreach my $matchingpoint (@$match){

    $record_a->field('005')->data && $record_b->field('005')->data &&

	  my $string = build_simplequery($matchingpoint,$record);

    $record_a->field('005')->data >= $record_b->field('005')->data;

	  push @searchstrings,$string if (length($string)>0);

}

sub GetRecordId {

    my $marcrecord = shift;

    my $tag = shift;

    my $subfield = shift;

    if ($tag lt "010") {

        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);

    }

    elsif ($subfield) {

        if ($marcrecord->field($tag)) {

            return $marcrecord->subfield($tag, $subfield);

    return join(" $op ",@searchstrings);

    return join(" $op ", @searchstrings);

	my $element=shift;

    my ($matchpoint, $record) = @_;

	my $record=shift;

    my ($index,$recorddata)=split /,/,$element;

    my ($index, $record_data) = split (/,/, $matchpoint);

    if ($recorddata=~/(\d{3})(.*)/) {

    if ($record_data =~ /(\d{3})(.*)/) {

        my ($tag,$subfields) =($1,$2);

        my ($tag, $subfields) = ($1, $2);

        foreach my $field ($record->field($tag)){

        foreach my $field ($record->field($tag)) {

		  if (length($field->as_string("$subfields"))>0){

            if (length($field->as_string("$subfields")) > 0) {

              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";

                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");

		  }

            }

    return join(" $op ",@searchstrings);

    return join(" $op ", @searchstrings);

sub printlog{

sub printlog {

	my $logelements=shift;

    my $logelements = shift;

    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";

    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";

sub get_heading_fields{

sub get_heading_fields {

    if ($authtypes){

    if ($authtypes) {

        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};

        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };

    unless ($headingfields){

    unless ($headingfields) {

        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});

        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});

        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};

        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };