View | Details | Raw Unified | Return to bug 37020
Collapse All | Expand All

(-)a/misc/migration_tools/bulkmarcimport.pl (-303 / +301 lines)
Lines 319-330 if ($logfile) { Link Here
319
    print $loghandle "id;operation;status\n";
319
    print $loghandle "id;operation;status\n";
320
}
320
}
321
321
322
my $record_number = 0;
322
my $record_number     = 0;
323
my $logger        = Koha::Logger->get;
323
my $records_exhausted = 0;
324
my $schema        = Koha::Database->schema;
324
my $logger            = Koha::Logger->get;
325
my $marc_records  = [];
325
my $schema            = Koha::Database->schema;
326
my $lint          = MARC::Lint->new;
326
my $lint              = MARC::Lint->new;
327
RECORD: while () {
327
328
329
while () {
330
328
    my $record;
331
    my $record;
329
    $record_number++;
332
    $record_number++;
330
333
Lines 342-347 RECORD: while () { Link Here
342
        next;
345
        next;
343
    }
346
    }
344
    if ($record) {
347
    if ($record) {
348
        $record_number++;
345
349
346
        if ($skip_bad_records) {
350
        if ($skip_bad_records) {
347
            my $xml = $record->as_xml_record();
351
            my $xml = $record->as_xml_record();
Lines 371-736 RECORD: while () { Link Here
371
        }
375
        }
372
        SetUTF8Flag($record);
376
        SetUTF8Flag($record);
373
        &$localcust($record) if $localcust;
377
        &$localcust($record) if $localcust;
374
        push @{$marc_records}, $record;
375
    } else {
376
        last;
377
    }
378
}
379
380
$record_number = 0;
381
my $records_total = @{$marc_records};
382
$schema->txn_begin;
383
RECORD: foreach my $record ( @{$marc_records} ) {
384
    $record_number++;
385
    if ( ( $verbose // 1 ) == 1 ) {    #no dot for verbose==2
386
        print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' );
387
    }
388
378
389
    if ( $marc_mod_template_id > 0 ) {
390
        print "Modifying MARC\n" if $verbose;
391
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
392
    }
393
379
394
    my $isbn;
380
        if ( ( $verbose // 1 ) == 1 ) {    #no dot for verbose==2
381
            print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' );
382
        }
395
383
396
    # remove trailing - in isbn (only for biblios, of course)
384
        if ( $marc_mod_template_id > 0 ) {
397
    if ( $biblios && ( $cleanisbn || $isbn_check ) ) {
385
            print "Modifying MARC\n" if $verbose;
398
        my $tag   = $marc_flavour eq 'UNIMARC' ? '010' : '020';
386
            ModifyRecordWithTemplate( $marc_mod_template_id, $record );
399
        my $field = $record->field($tag);
400
        $isbn = $field && $field->subfield('a');
401
        if ( $isbn && $cleanisbn ) {
402
            $isbn =~ s/-//g;
403
            $field->update( 'a' => $isbn );
404
        }
387
        }
405
    }
406
388
407
    # search for duplicates (based on Local-number)
389
        my $isbn;
408
    my $originalid        = GetRecordId( $record, $tagid, $subfieldid );
390
409
    my $matched_record_id = undef;
391
        # remove trailing - in isbn (only for biblios, of course)
410
    if ($match) {
392
        if ( $biblios && ( $cleanisbn || $isbn_check ) ) {
411
        require C4::Search;
393
            my $tag   = $marc_flavour eq 'UNIMARC' ? '010' : '020';
412
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
394
            my $field = $record->field($tag);
413
        my $query  = build_query( $match, $record );
395
            $isbn = $field && $field->subfield('a');
414
        $logger->debug("Bulkmarcimport: $query");
396
            if ( $isbn && $cleanisbn ) {
415
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
397
                $isbn =~ s/-//g;
416
398
                $field->update( 'a' => $isbn );
417
        # changed to warn so able to continue with one broken record
399
            }
418
        if ( defined $error ) {
419
            warn "unable to search the database for duplicates : $error";
420
            printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile);
421
            next RECORD;
422
        }
400
        }
423
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
401
424
402
        # search for duplicates (based on Local-number)
425
        # sub SimpleSearch could return undefined, but only on error, so
403
        my $originalid        = GetRecordId( $record, $tagid, $subfieldid );
426
        # should not really need to safeguard here, but do so anyway
404
        my $matched_record_id = undef;
427
        $results //= [];
405
        if ($match) {
428
        if ( @{$results} == 1 ) {
406
            require C4::Search;
429
            my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] );
407
            my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
430
            SetUTF8Flag($matched_record);
408
            my $query  = build_query( $match, $record );
431
            $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid );
409
            $logger->debug("Bulkmarcimport: $query");
432
410
            my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
433
            if ( $authorities && $marc_flavour ) {
411
434
412
            # changed to warn so able to continue with one broken record
435
                #Skip if authority in database is the same or newer than the incoming record
413
            if ( defined $error ) {
436
                if ( RecordRevisionIsGtOrEq( $matched_record, $record ) ) {
414
                warn "unable to search the database for duplicates : $error";
437
                    if ($yamlfile) {
415
                printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile);
438
                        $yamlhash->{$originalid} = YAMLFileEntry(
416
                next RECORD;
439
                            $matched_record,
417
            }
440
                            $matched_record_id,
418
            $logger->debug("Bulkmarcimport: $query $server : $totalhits");
441
                            0
419
442
                        );
420
            # sub SimpleSearch could return undefined, but only on error, so
421
            # should not really need to safeguard here, but do so anyway
422
            $results //= [];
423
            if ( @{$results} == 1 ) {
424
                my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] );
425
                SetUTF8Flag($matched_record);
426
                $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid );
427
428
                if ( $authorities && $marc_flavour ) {
429
430
                    #Skip if authority in database is the same or newer than the incoming record
431
                    if ( RecordRevisionIsGtOrEq( $matched_record, $record ) ) {
432
                        if ($yamlfile) {
433
                            $yamlhash->{$originalid} = YAMLFileEntry(
434
                                $matched_record,
435
                                $matched_record_id,
436
                                0
437
                            );
438
                        }
439
                        next;
443
                    }
440
                    }
444
                    next;
445
                }
441
                }
442
            } elsif ( @{$results} > 1 ) {
443
                $logger->debug("More than one match for: $query");
444
                next;
445
            } else {
446
                $logger->debug("No match for: $query");
446
            }
447
            }
447
        } elsif ( @{$results} > 1 ) {
448
            $logger->debug("More than one match for: $query");
449
            next;
450
        } else {
451
            $logger->debug("No match for: $query");
452
        }
453
448
454
        if ( $keepids && $originalid ) {
449
            if ( $keepids && $originalid ) {
455
            my $storeidfield;
450
                my $storeidfield;
456
            if ( length($keepids) == 3 ) {
451
                if ( length($keepids) == 3 ) {
457
                $storeidfield = MARC::Field->new( $keepids, $originalid );
452
                    $storeidfield = MARC::Field->new( $keepids, $originalid );
458
            } else {
453
                } else {
459
                $storeidfield =
454
                    $storeidfield =
460
                    MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
455
                        MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
456
                }
457
                $record->insert_fields_ordered($storeidfield);
458
                $record->delete_field( $record->field($tagid) );
461
            }
459
            }
462
            $record->insert_fields_ordered($storeidfield);
463
            $record->delete_field( $record->field($tagid) );
464
        }
460
        }
465
    }
466
461
467
    foreach my $stringfilter (@$filters) {
462
        foreach my $stringfilter (@$filters) {
468
        if ( length($stringfilter) == 3 ) {
463
            if ( length($stringfilter) == 3 ) {
469
            foreach my $field ( $record->field($stringfilter) ) {
464
                foreach my $field ( $record->field($stringfilter) ) {
470
                $record->delete_field($field);
465
                    $record->delete_field($field);
471
                $logger->debug( "Removed: ", $field->as_string );
466
                    $logger->debug( "Removed: ", $field->as_string );
472
            }
467
                }
473
        } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) {
468
            } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) {
474
            my $removetag      = $1;
469
                my $removetag      = $1;
475
            my $removesubfield = $2;
470
                my $removesubfield = $2;
476
            my $removematch    = $3;
471
                my $removematch    = $3;
477
            if ( ( $removetag > "010" ) && $removesubfield ) {
472
                if ( ( $removetag > "010" ) && $removesubfield ) {
478
                foreach my $field ( $record->field($removetag) ) {
473
                    foreach my $field ( $record->field($removetag) ) {
479
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
474
                        $field->delete_subfield( code => "$removesubfield", match => $removematch );
480
                    $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) );
475
                        $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) );
476
                    }
481
                }
477
                }
482
            }
478
            }
483
        }
479
        }
484
    }
480
        unless ($test_parameter) {
485
    unless ($test_parameter) {
481
            $schema->txn_begin;
486
        if ($authorities) {
482
487
            my $authtypecode = GuessAuthTypeCode( $record, $heading_fields );
483
            if ($authorities) {
488
            my $authid;
484
                my $authtypecode = GuessAuthTypeCode( $record, $heading_fields );
489
485
                my $authid;
490
            if ($matched_record_id) {
486
491
                if ($update) {
487
                if ($matched_record_id) {
492
                    ## Authority has an id and is in database: update
488
                    if ($update) {
493
                    eval {
489
                        ## Authority has an id and is in database: update
494
                        ($authid) = ModAuthority(
490
                        eval {
495
                            $matched_record_id, $record, $authtypecode,
491
                            ($authid) = ModAuthority(
496
                            $mod_authority_options,
492
                                $matched_record_id, $record, $authtypecode,
493
                                $mod_authority_options,
494
                            );
495
                        };
496
                        if ($@) {
497
                            warn "ERROR: Update authority $matched_record_id failed: $@\n";
498
                            printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
499
                            next RECORD;
500
                        } else {
501
                            printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile);
502
                        }
503
                    } elsif ($logfile) {
504
                        warn "WARNING: Update authority $originalid skipped";
505
                        printlog(
506
                            {
507
                                id     => $matched_record_id,
508
                                op     => "update",
509
                                status =>
510
                                    "warning: authority already in database and option -update not enabled, skipping..."
511
                            }
497
                        );
512
                        );
498
                    };
513
                    }
514
                } elsif ($insert) {
515
                    ## An authid is defined but no authority in database: insert
516
                    eval { ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); };
499
                    if ($@) {
517
                    if ($@) {
500
                        warn "ERROR: Update authority $matched_record_id failed: $@\n";
518
                        warn "ERROR: Insert authority $originalid failed: $@\n";
501
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
519
                        printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
502
                        next RECORD;
520
                        next RECORD;
503
                    } else {
521
                    } else {
504
                        printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile);
522
                        printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile);
505
                    }
523
                    }
506
                } elsif ($logfile) {
524
                } else {
507
                    warn "WARNING: Update authority $originalid skipped";
525
                    warn "WARNING: Insert authority $originalid skipped";
508
                    printlog(
526
                    printlog(
509
                        {
527
                        {
510
                            id     => $matched_record_id,
528
                            id     => $originalid, op => "insert",
511
                            op     => "update",
529
                            status => "warning : authority not in database and option -insert not enabled, skipping..."
512
                            status =>
513
                                "warning: authority already in database and option -update not enabled, skipping..."
514
                        }
530
                        }
515
                    );
531
                    ) if ($logfile);
516
                }
532
                }
517
            } elsif ($insert) {
533
518
                ## An authid is defined but no authority in database: insert
534
                if ($yamlfile) {
519
                eval { ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); };
535
                    $yamlhash->{$originalid} = YAMLFileEntry(
520
                if ($@) {
536
                        $record,
521
                    warn "ERROR: Insert authority $originalid failed: $@\n";
537
                        $authid,
522
                    printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
538
                        1    #@FIXME: Really always updated?
523
                    next RECORD;
539
                    );
524
                } else {
525
                    printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile);
526
                }
540
                }
541
                push @search_engine_record_ids, $authid;
542
                push @search_engine_records,    $record;
527
            } else {
543
            } else {
528
                warn "WARNING: Insert authority $originalid skipped";
544
                my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id );
529
                printlog(
530
                    {
531
                        id     => $originalid, op => "insert",
532
                        status => "warning : authority not in database and option -insert not enabled, skipping..."
533
                    }
534
                ) if ($logfile);
535
            }
536
537
            if ($yamlfile) {
538
                $yamlhash->{$originalid} = YAMLFileEntry(
539
                    $record,
540
                    $authid,
541
                    1    #@FIXME: Really always updated?
542
                );
543
            }
544
            push @search_engine_record_ids, $authid;
545
            push @search_engine_records,    $record;
546
        } else {
547
            my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id );
548
545
549
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
546
                # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
550
            if ( !$matched_record_id && $isbn_check && $isbn ) {
547
                if ( !$matched_record_id && $isbn_check && $isbn ) {
551
                $sth_isbn->execute($isbn);
548
                    $sth_isbn->execute($isbn);
552
                ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow;
549
                    ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow;
553
            }
550
                }
554
551
555
            if ( defined $idmapfl && $matched_record_id ) {
552
                if ( defined $idmapfl && $matched_record_id ) {
556
                if ( $sourcetag < "010" ) {
553
                    if ( $sourcetag < "010" ) {
557
                    if ( $record->field($sourcetag) ) {
554
                        if ( $record->field($sourcetag) ) {
558
                        my $source = $record->field($sourcetag)->data();
555
                            my $source = $record->field($sourcetag)->data();
556
                            printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
557
                        }
558
                    } else {
559
                        my $source = $record->subfield( $sourcetag, $sourcesubfield );
559
                        printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
560
                        printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
560
                    }
561
                    }
561
                } else {
562
                    my $source = $record->subfield( $sourcetag, $sourcesubfield );
563
                    printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
564
                }
562
                }
565
            }
566
563
567
            # Create biblio, unless we already have it (either match or ISBN)
564
                # Create biblio, unless we already have it (either match or ISBN)
568
            if ($matched_record_id) {
565
                if ($matched_record_id) {
569
                eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; };
566
                    eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; };
570
                if ($update) {
567
                    if ($update) {
571
                    my $success;
568
                        my $success;
572
                    eval {
569
                        eval {
573
                        $success = ModBiblio(
570
                            $success = ModBiblio(
574
                            $record, $matched_record_id, GetFrameworkCode($matched_record_id),
571
                                $record, $matched_record_id, GetFrameworkCode($matched_record_id),
575
                            $mod_biblio_options
572
                                $mod_biblio_options
576
                        );
573
                            );
577
                    };
574
                        };
575
                        if ($@) {
576
                            warn "ERROR: Update biblio $matched_record_id failed: $@\n";
577
                            printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
578
                            next RECORD;
579
                        } elsif ( !$success ) {
580
                            warn "ERROR: Update biblio $matched_record_id failed for unknown reason";
581
                            printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
582
                            next RECORD;
583
                        } else {
584
                            $record_id = $matched_record_id;
585
                            printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
586
                        }
587
                    } else {
588
                        warn "WARNING: Update biblio $originalid skipped";
589
                        printlog(
590
                            {
591
                                id     => $matched_record_id, op => "update",
592
                                status => "warning : already in database and option -update not enabled, skipping..."
593
                            }
594
                        ) if ($logfile);
595
                    }
596
                } elsif ($insert) {
597
                    my $record_clone = $record->clone();
598
                    C4::Biblio::_strip_item_fields($record_clone);
599
                    eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) };
578
                    if ($@) {
600
                    if ($@) {
579
                        warn "ERROR: Update biblio $matched_record_id failed: $@\n";
601
                        warn "ERROR: Insert biblio $originalid failed: $@\n";
580
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
602
                        printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
581
                        next RECORD;
582
                    } elsif ( !$success ) {
583
                        warn "ERROR: Update biblio $matched_record_id failed for unknown reason";
584
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
585
                        next RECORD;
603
                        next RECORD;
586
                    } else {
604
                    } else {
587
                        $record_id = $matched_record_id;
605
                        printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile);
588
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
589
                    }
606
                    }
607
608
                    # If incoming record has bib ids set we need to transfer
609
                    # new ids from record_clone to incoming record to avoid
610
                    # working on wrong record (the original record) later on
611
                    # when adding items for example
612
                    C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber );
590
                } else {
613
                } else {
591
                    warn "WARNING: Update biblio $originalid skipped";
614
                    warn "WARNING: Insert biblio $originalid skipped";
592
                    printlog(
615
                    printlog(
593
                        {
616
                        {
594
                            id     => $matched_record_id, op => "update",
617
                            id     => $originalid, op => "insert",
595
                            status => "warning : already in database and option -update not enabled, skipping..."
618
                            status => "warning : biblio not in database and option -insert not enabled, skipping..."
596
                        }
619
                        }
597
                    ) if ($logfile);
620
                    ) if ($logfile);
598
                }
599
            } elsif ($insert) {
600
                my $record_clone = $record->clone();
601
                C4::Biblio::_strip_item_fields($record_clone);
602
                eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) };
603
                if ($@) {
604
                    warn "ERROR: Insert biblio $originalid failed: $@\n";
605
                    printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
606
                    next RECORD;
621
                    next RECORD;
607
                } else {
608
                    printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile);
609
                }
622
                }
610
623
                my $record_has_added_items = 0;
611
                # If incoming record has bib ids set we need to transfer
624
                if ($record_id) {
612
                # new ids from record_clone to incoming record to avoid
625
                    $yamlhash->{$originalid} = $record_id if $yamlfile;
613
                # working on wrong record (the original record) later on
614
                # when adding items for example
615
                C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber );
616
            } else {
617
                warn "WARNING: Insert biblio $originalid skipped";
618
                printlog(
619
                    {
620
                        id     => $originalid, op => "insert",
621
                        status => "warning : biblio not in database and option -insert not enabled, skipping..."
622
                    }
623
                ) if ($logfile);
624
                next RECORD;
625
            }
626
            my $record_has_added_items = 0;
627
            if ($record_id) {
628
                $yamlhash->{$originalid} = $record_id if $yamlfile;
629
                eval {
630
                    ( $itemnumbers_ref, $errors_ref ) =
631
                        AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework );
632
                };
633
                my $error_adding = $@;
634
635
                if ($error_adding) {
636
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
637
                    printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
638
639
                    # if we failed because of an exception, assume that
640
                    # the MARC columns in biblioitems were not set.
641
                    next RECORD;
642
                }
643
644
                $record_has_added_items = @{$itemnumbers_ref};
645
646
                if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' }
647
                    @$errors_ref )
648
                {
649
                    # Find the record called 'barcode'
650
                    my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode');
651
652
                    # Now remove any items that didn't have a duplicate_barcode error,
653
                    # erase the barcodes on items that did, and re-add those items.
654
                    my %dupes;
655
                    foreach my $i ( 0 .. $#{$errors_ref} ) {
656
                        my $ref = $errors_ref->[$i];
657
                        if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) {
658
                            $dupes{ $ref->{item_sequence} } = 1;
659
660
                            # Delete the error message because we're going to
661
                            # retry this one.
662
                            delete $errors_ref->[$i];
663
                        }
664
                    }
665
                    my $seq = 0;
666
                    foreach my $field ( $record->field($tag) ) {
667
                        $seq++;
668
                        if ( $dupes{$seq} ) {
669
670
                            # Here we remove the barcode
671
                            $field->delete_subfield( code => $sub );
672
                        } else {
673
674
                            # otherwise we delete the field because we don't want
675
                            # two of them
676
                            $record->delete_fields($field);
677
                        }
678
                    }
679
680
                    # Now re-add the record as before, adding errors to the prev list
681
                    my $more_errors;
682
                    eval {
626
                    eval {
683
                        ( $itemnumbers_ref, $more_errors ) =
627
                        ( $itemnumbers_ref, $errors_ref ) =
684
                            AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' );
628
                            AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework );
685
                    };
629
                    };
686
                    if ($@) {
630
                    my $error_adding = $@;
687
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
631
632
                    if ($error_adding) {
633
                        warn "ERROR: Adding items to bib $record_id failed: $error_adding";
688
                        printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
634
                        printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
689
635
690
                        # if we failed because of an exception, assume that
636
                        # if we failed because of an exception, assume that
691
                        # the MARC columns in biblioitems were not set.
637
                        # the MARC columns in biblioitems were not set.
692
                        next RECORD;
638
                        next RECORD;
693
                    }
639
                    }
694
                    $record_has_added_items ||= @{$itemnumbers_ref};
640
695
                    if ( @{$more_errors} ) {
641
                    $record_has_added_items = @{$itemnumbers_ref};
696
                        push @$errors_ref, @{$more_errors};
642
643
                    if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' }
644
                        @$errors_ref )
645
                    {
646
                        # Find the record called 'barcode'
647
                        my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode');
648
649
                        # Now remove any items that didn't have a duplicate_barcode error,
650
                        # erase the barcodes on items that did, and re-add those items.
651
                        my %dupes;
652
                        foreach my $i ( 0 .. $#{$errors_ref} ) {
653
                            my $ref = $errors_ref->[$i];
654
                            if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) {
655
                                $dupes{ $ref->{item_sequence} } = 1;
656
657
                                # Delete the error message because we're going to
658
                                # retry this one.
659
                                delete $errors_ref->[$i];
660
                            }
661
                        }
662
                        my $seq = 0;
663
                        foreach my $field ( $record->field($tag) ) {
664
                            $seq++;
665
                            if ( $dupes{$seq} ) {
666
667
                                # Here we remove the barcode
668
                                $field->delete_subfield( code => $sub );
669
                            } else {
670
671
                                # otherwise we delete the field because we don't want
672
                                # two of them
673
                                $record->delete_fields($field);
674
                            }
675
                        }
676
677
                        # Now re-add the record as before, adding errors to the prev list
678
                        my $more_errors;
679
                        eval {
680
                            ( $itemnumbers_ref, $more_errors ) =
681
                                AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' );
682
                        };
683
                        if ($@) {
684
                            warn "ERROR: Adding items to bib $record_id failed: $@\n";
685
                            printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
686
687
                            # if we failed because of an exception, assume that
688
                            # the MARC columns in biblioitems were not set.
689
                            next RECORD;
690
                        }
691
                        $record_has_added_items ||= @{$itemnumbers_ref};
692
                        if ( @{$more_errors} ) {
693
                            push @$errors_ref, @{$more_errors};
694
                        }
697
                    }
695
                    }
698
                }
699
696
700
                if ($record_has_added_items) {
697
                    if ($record_has_added_items) {
701
                    printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile);
698
                        printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile);
702
                }
699
                    }
703
700
704
                if ( @{$errors_ref} ) {
701
                    if ( @{$errors_ref} ) {
705
                    report_item_errors( $record_id, $errors_ref );
702
                        report_item_errors( $record_id, $errors_ref );
706
                }
703
                    }
707
704
708
                my $biblio = Koha::Biblios->find($record_id);
705
                    my $biblio = Koha::Biblios->find($record_id);
709
                $record = $biblio->metadata_record( { embed_items => 1 } );
706
                    $record = $biblio->metadata_record( { embed_items => 1 } );
710
707
711
                push @search_engine_record_ids, $record_id;
708
                    push @search_engine_record_ids, $record_id;
712
                push @search_engine_records,    $record;
709
                    push @search_engine_records,    $record;
710
                }
713
            }
711
            }
714
        }
712
        }
715
        if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) {
713
        print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2;
716
            $schema->txn_commit;
714
    } else {
717
            $schema->txn_begin;
715
        $records_exhausted = 1;
718
            if ($indexer) {
716
    }
719
                $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing;
717
720
                if ( C4::Context->preference('AutoLinkBiblios') ) {
718
    if ( $record_number % $commitnum == 0 || $record_number == $number || $records_exhausted ) {
721
                    foreach my $record (@search_engine_records) {
719
        if ($indexer) {
722
                        BiblioAutoLink( $record, $framework );
720
            $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing;
723
                    }
721
            if ( C4::Context->preference('AutoLinkBiblios') ) {
722
                foreach my $record (@search_engine_records) {
723
                    BiblioAutoLink( $record, $framework );
724
                }
724
                }
725
                @search_engine_record_ids = ();
726
                @search_engine_records    = ();
727
            }
725
            }
726
            @search_engine_record_ids = ();
727
            @search_engine_records    = ();
728
        }
728
        }
729
        $schema->txn_commit;
729
    }
730
    }
730
    print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2;
731
    last if $record_number == $number || $records_exhausted;
731
    last                                 if $record_number == $number;
732
}
732
}
733
$schema->txn_commit;
734
733
735
if ($fk_off) {
734
if ($fk_off) {
736
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
735
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
737
- 

Return to bug 37020