View | Details | Raw Unified | Return to bug 7475
Collapse All | Expand All

(-)a/C4/ImportBatch.pm (-101 / +219 lines)
Lines 1-6 Link Here
1
package C4::ImportBatch;
1
package C4::ImportBatch;
2
2
3
# Copyright (C) 2007 LibLime
3
# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 40-53 BEGIN { Link Here
40
    GetImportRecordMarcXML
40
    GetImportRecordMarcXML
41
    AddImportBatch
41
    AddImportBatch
42
    GetImportBatch
42
    GetImportBatch
43
    AddAuthorityToBatch
43
    AddBiblioToBatch
44
    AddBiblioToBatch
44
    AddItemsToImportBiblio
45
    AddItemsToImportBiblio
46
    ModAuthorityInBatch
45
    ModBiblioInBatch
47
    ModBiblioInBatch
46
48
47
    BatchStageMarcRecords
49
    BatchStageMarcRecords
48
    BatchFindBibDuplicates
50
    BatchFindDuplicates
49
    BatchCommitBibRecords
51
    BatchCommitRecords
50
    BatchRevertBibRecords
52
    BatchRevertRecords
51
    CleanBatch
53
    CleanBatch
52
54
53
    GetAllImportBatches
55
    GetAllImportBatches
Lines 272-281 sub ModBiblioInBatch { Link Here
272
274
273
}
275
}
274
276
277
=head2 AddAuthToBatch
278
279
  my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
280
                $marc_record, $encoding, $z3950random, $update_counts);
281
282
=cut
283
284
sub AddAuthToBatch {
285
    my $batch_id = shift;
286
    my $record_sequence = shift;
287
    my $marc_record = shift;
288
    my $encoding = shift;
289
    my $z3950random = shift;
290
    my $update_counts = @_ ? shift : 1;
291
292
    my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random);
293
    _add_auth_fields($import_record_id, $marc_record);
294
    _update_batch_record_counts($batch_id) if $update_counts;
295
    return $import_record_id;
296
}
297
298
=head2 ModAuthInBatch
299
300
  ModAuthInBatch($import_record_id, $marc_record);
301
302
=cut
303
304
sub ModAuthInBatch {
305
    my ($import_record_id, $marc_record) = @_;
306
307
    _update_import_record_marc($import_record_id, $marc_record);
308
309
}
310
275
=head2 BatchStageMarcRecords
311
=head2 BatchStageMarcRecords
276
312
277
  ($batch_id, $num_records, $num_items, @invalid_records) = 
313
  ($batch_id, $num_records, $num_items, @invalid_records) = 
278
    BatchStageMarcRecords($encoding, $marc_records, $file_name, 
314
    BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name,
279
                          $comments, $branch_code, $parse_items,
315
                          $comments, $branch_code, $parse_items,
280
                          $leave_as_staging, 
316
                          $leave_as_staging, 
281
                          $progress_interval, $progress_callback);
317
                          $progress_interval, $progress_callback);
Lines 283-288 sub ModBiblioInBatch { Link Here
283
=cut
319
=cut
284
320
285
sub  BatchStageMarcRecords {
321
sub  BatchStageMarcRecords {
322
    my $record_type = shift;
286
    my $encoding = shift;
323
    my $encoding = shift;
287
    my $marc_records = shift;
324
    my $marc_records = shift;
288
    my $file_name = shift;
325
    my $file_name = shift;
Lines 338-347 sub BatchStageMarcRecords { Link Here
338
            push @invalid_records, $marc_blob;
375
            push @invalid_records, $marc_blob;
339
        } else {
376
        } else {
340
            $num_valid++;
377
            $num_valid++;
341
            $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
378
            if ($record_type eq 'biblio') {
342
            if ($parse_items) {
379
                $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
343
                my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
380
                if ($parse_items) {
344
                $num_items += scalar(@import_items_ids);
381
                    my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
382
                    $num_items += scalar(@import_items_ids);
383
                }
384
            } elsif ($record_type eq 'auth') {
385
                $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
345
            }
386
            }
346
        }
387
        }
347
    }
388
    }
Lines 392-400 sub AddItemsToImportBiblio { Link Here
392
    return @import_items_ids;
433
    return @import_items_ids;
393
}
434
}
394
435
395
=head2 BatchFindBibDuplicates
436
=head2 BatchFindDuplicates
396
437
397
  my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 
438
  my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
398
             $max_matches, $progress_interval, $progress_callback);
439
             $max_matches, $progress_interval, $progress_callback);
399
440
400
Goes through the records loaded in the batch and attempts to 
441
Goes through the records loaded in the batch and attempts to 
Lines 412-418 singular argument. Link Here
412
453
413
=cut
454
=cut
414
455
415
sub BatchFindBibDuplicates {
456
sub BatchFindDuplicates {
416
    my $batch_id = shift;
457
    my $batch_id = shift;
417
    my $matcher = shift;
458
    my $matcher = shift;
418
    my $max_matches = @_ ? shift : 10;
459
    my $max_matches = @_ ? shift : 10;
Lines 430-438 sub BatchFindBibDuplicates { Link Here
430
471
431
    my $dbh = C4::Context->dbh;
472
    my $dbh = C4::Context->dbh;
432
473
433
    my $sth = $dbh->prepare("SELECT import_record_id, marc
474
    my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, marc
434
                             FROM import_records
475
                             FROM import_records
435
                             JOIN import_biblios USING (import_record_id)
476
                             LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
477
                             LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
436
                             WHERE import_batch_id = ?");
478
                             WHERE import_batch_id = ?");
437
    $sth->execute($batch_id);
479
    $sth->execute($batch_id);
438
    my $num_with_matches = 0;
480
    my $num_with_matches = 0;
Lines 460-474 sub BatchFindBibDuplicates { Link Here
460
    return $num_with_matches;
502
    return $num_with_matches;
461
}
503
}
462
504
463
=head2 BatchCommitBibRecords
505
=head2 BatchCommitRecords
464
506
465
  my ($num_added, $num_updated, $num_items_added, $num_items_errored, 
507
  my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
466
      $num_ignored) = BatchCommitBibRecords($batch_id, $framework,
508
        BatchCommitRecords($batch_id, $framework,
467
                      $progress_interval, $progress_callback);
509
        $progress_interval, $progress_callback);
468
510
469
=cut
511
=cut
470
512
471
sub BatchCommitBibRecords {
513
sub BatchCommitRecords {
472
    my $batch_id = shift;
514
    my $batch_id = shift;
473
    my $framework = shift;
515
    my $framework = shift;
474
516
Lines 483-507 sub BatchCommitBibRecords { Link Here
483
        $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
525
        $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
484
    }
526
    }
485
527
528
    my $record_type;
486
    my $num_added = 0;
529
    my $num_added = 0;
487
    my $num_updated = 0;
530
    my $num_updated = 0;
488
    my $num_items_added = 0;
531
    my $num_items_added = 0;
489
    my $num_items_errored = 0;
532
    my $num_items_errored = 0;
490
    my $num_ignored = 0;
533
    my $num_ignored = 0;
491
    # commit (i.e., save, all records in the batch)
534
    # commit (i.e., save, all records in the batch)
492
    # FIXME biblio only at the moment
493
    SetImportBatchStatus('importing');
535
    SetImportBatchStatus('importing');
494
    my $overlay_action = GetImportBatchOverlayAction($batch_id);
536
    my $overlay_action = GetImportBatchOverlayAction($batch_id);
495
    my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
537
    my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
496
    my $item_action = GetImportBatchItemAction($batch_id);
538
    my $item_action = GetImportBatchItemAction($batch_id);
539
    my $item_tag;
540
    my $item_subfield;
497
    my $dbh = C4::Context->dbh;
541
    my $dbh = C4::Context->dbh;
498
    my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding
542
    my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
499
                             FROM import_records
543
                             FROM import_records
500
                             JOIN import_biblios USING (import_record_id)
544
                             LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
545
                             LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
501
                             WHERE import_batch_id = ?");
546
                             WHERE import_batch_id = ?");
502
    $sth->execute($batch_id);
547
    $sth->execute($batch_id);
503
    my $rec_num = 0;
548
    my $rec_num = 0;
504
    while (my $rowref = $sth->fetchrow_hashref) {
549
    while (my $rowref = $sth->fetchrow_hashref) {
550
        $record_type = $rowref->{'record_type'};
505
        $rec_num++;
551
        $rec_num++;
506
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
552
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
507
            &$progress_callback($rec_num);
553
            &$progress_callback($rec_num);
Lines 513-579 sub BatchCommitBibRecords { Link Here
513
559
514
        my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
560
        my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
515
561
516
        # remove any item tags - rely on BatchCommitItems
562
        if ($record_type eq 'biblio') {
517
        my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
563
            # remove any item tags - rely on BatchCommitItems
518
        foreach my $item_field ($marc_record->field($item_tag)) {
564
            ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
519
            $marc_record->delete_field($item_field);
565
            foreach my $item_field ($marc_record->field($item_tag)) {
566
                $marc_record->delete_field($item_field);
567
            }
520
        }
568
        }
521
569
522
        # decide what what to do with the bib and item records
570
        my ($record_result, $item_result, $record_match) =
523
        my ($bib_result, $item_result, $bib_match) = 
524
            _get_commit_action($overlay_action, $nomatch_action, $item_action, 
571
            _get_commit_action($overlay_action, $nomatch_action, $item_action, 
525
                               $rowref->{'overlay_status'}, $rowref->{'import_record_id'});
572
                               $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
526
573
527
        if ($bib_result eq 'create_new') {
574
        my $recordid;
575
        my $query;
576
        if ($record_result eq 'create_new') {
528
            $num_added++;
577
            $num_added++;
529
            my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, $framework);
578
            if ($record_type eq 'biblio') {
530
            my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
579
                my $biblioitemnumber;
531
            $sth->execute($biblionumber, $rowref->{'import_record_id'});
580
                ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
532
            $sth->finish();
581
                $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
533
            if ($item_result eq 'create_new') {
582
                if ($item_result eq 'create_new') {
534
                my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
583
                    my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
535
                $num_items_added += $bib_items_added;
584
                    $num_items_added += $bib_items_added;
536
                $num_items_errored += $bib_items_errored;
585
                    $num_items_errored += $bib_items_errored;
586
                }
587
            } else {
588
                my $authid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
589
                $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
537
            }
590
            }
591
            my $sth = $dbh->prepare_cached($query);
592
            $sth->execute($recordid, $rowref->{'import_record_id'});
593
            $sth->finish();
538
            SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
594
            SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
539
        } elsif ($bib_result eq 'replace') {
595
        } elsif ($record_result eq 'replace') {
540
            $num_updated++;
596
            $num_updated++;
541
            my $biblionumber = $bib_match;
597
            $recordid = $record_match;
542
            my ($count, $oldbiblio) = GetBiblio($biblionumber);
598
            my $oldxml;
543
            my $oldxml = GetXmlBiblio($biblionumber);
599
            if ($record_type eq 'biblio') {
544
600
                my ($count, $oldbiblio) = GetBiblio($recordid);
545
            # remove item fields so that they don't get
601
                $oldxml = GetXmlBiblio($recordid);
546
            # added again if record is reverted
602
547
            my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'});
603
                # remove item fields so that they don't get
548
            foreach my $item_field ($old_marc->field($item_tag)) {
604
                # added again if record is reverted
549
                $old_marc->delete_field($item_field);
605
                my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'});
550
            }
606
                foreach my $item_field ($old_marc->field($item_tag)) {
607
                    $old_marc->delete_field($item_field);
608
                }
609
                $oldxml = $old_marc->as_xml();
610
611
                ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
612
                $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
613
614
                if ($item_result eq 'create_new') {
615
                    my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
616
                    $num_items_added += $bib_items_added;
617
                    $num_items_errored += $bib_items_errored;
618
                }
619
            } else {
620
                my $oldxml = GetAuthorityXML($recordid);
551
621
552
            ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'});
622
                ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
623
                $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
624
            }
553
            my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
625
            my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
554
            $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'});
626
            $sth->execute($oldxml, $rowref->{'import_record_id'});
555
            $sth->finish();
627
            $sth->finish();
556
            my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
628
            my $sth2 = $dbh->prepare_cached($query);
557
            $sth2->execute($biblionumber, $rowref->{'import_record_id'});
629
            $sth2->execute($recordid, $rowref->{'import_record_id'});
558
            $sth2->finish();
630
            $sth2->finish();
559
            if ($item_result eq 'create_new') {
560
                my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
561
                $num_items_added += $bib_items_added;
562
                $num_items_errored += $bib_items_errored;
563
            }
564
            SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
631
            SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
565
            SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
632
            SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
566
        } elsif ($bib_result eq 'ignore') {
633
        } elsif ($record_result eq 'ignore') {
567
            $num_ignored++;
634
            $num_ignored++;
568
            my $biblionumber = $bib_match;
635
            if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') {
569
            if (defined $biblionumber and $item_result eq 'create_new') {
636
                my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
570
                my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
571
                $num_items_added += $bib_items_added;
637
                $num_items_added += $bib_items_added;
572
                $num_items_errored += $bib_items_errored;
638
                $num_items_errored += $bib_items_errored;
573
                # still need to record the matched biblionumber so that the
639
                # still need to record the matched biblionumber so that the
574
                # items can be reverted
640
                # items can be reverted
575
                my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
641
                my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
576
                $sth2->execute($biblionumber, $rowref->{'import_record_id'});
642
                $sth2->execute($recordid, $rowref->{'import_record_id'});
577
                SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
643
                SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
578
            }
644
            }
579
            SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
645
            SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
Lines 632-694 sub BatchCommitItems { Link Here
632
    return ($num_items_added, $num_items_errored);
698
    return ($num_items_added, $num_items_errored);
633
}
699
}
634
700
635
=head2 BatchRevertBibRecords
701
=head2 BatchRevertRecords
636
702
637
  my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, 
703
  my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, 
638
      $num_ignored) = BatchRevertBibRecords($batch_id);
704
      $num_ignored) = BatchRevertRecords($batch_id);
639
705
640
=cut
706
=cut
641
707
642
sub BatchRevertBibRecords {
708
sub BatchRevertRecords {
643
    my $batch_id = shift;
709
    my $batch_id = shift;
644
710
711
    my $record_type;
645
    my $num_deleted = 0;
712
    my $num_deleted = 0;
646
    my $num_errors = 0;
713
    my $num_errors = 0;
647
    my $num_reverted = 0;
714
    my $num_reverted = 0;
648
    my $num_items_deleted = 0;
649
    my $num_ignored = 0;
715
    my $num_ignored = 0;
716
    my $num_items_deleted = 0;
650
    # commit (i.e., save, all records in the batch)
717
    # commit (i.e., save, all records in the batch)
651
    # FIXME biblio only at the moment
652
    SetImportBatchStatus('reverting');
718
    SetImportBatchStatus('reverting');
653
    my $overlay_action = GetImportBatchOverlayAction($batch_id);
719
    my $overlay_action = GetImportBatchOverlayAction($batch_id);
654
    my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
720
    my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
655
    my $dbh = C4::Context->dbh;
721
    my $dbh = C4::Context->dbh;
656
    my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber
722
    my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
657
                             FROM import_records
723
                             FROM import_records
658
                             JOIN import_biblios USING (import_record_id)
724
                             LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
725
                             LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
659
                             WHERE import_batch_id = ?");
726
                             WHERE import_batch_id = ?");
660
    $sth->execute($batch_id);
727
    $sth->execute($batch_id);
661
    while (my $rowref = $sth->fetchrow_hashref) {
728
    while (my $rowref = $sth->fetchrow_hashref) {
729
        $record_type = $rowref->{'record_type'};
662
        if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
730
        if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
663
            $num_ignored++;
731
            $num_ignored++;
664
            next;
732
            next;
665
        }
733
        }
666
734
667
        my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
735
        my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
668
736
669
        if ($bib_result eq 'delete') {
737
        if ($record_result eq 'delete') {
670
            $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
738
            my $error = undef;
671
            my $error = DelBiblio($rowref->{'matched_biblionumber'});
739
            if  ($record_type eq 'biblio') {
740
                $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
741
                $error = DelBiblio($rowref->{'matched_biblionumber'});
742
            } else {
743
                my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
744
            }
672
            if (defined $error) {
745
            if (defined $error) {
673
                $num_errors++;
746
                $num_errors++;
674
            } else {
747
            } else {
675
                $num_deleted++;
748
                $num_deleted++;
676
                SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
749
                SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
677
            }
750
            }
678
        } elsif ($bib_result eq 'restore') {
751
        } elsif ($record_result eq 'restore') {
679
            $num_reverted++;
752
            $num_reverted++;
680
            my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'});
753
            my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'});
681
            my $biblionumber = $rowref->{'matched_biblionumber'};
754
            if ($record_type eq 'biblio') {
682
            my ($count, $oldbiblio) = GetBiblio($biblionumber);
755
                my $biblionumber = $rowref->{'matched_biblionumber'};
683
            $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
756
                my ($count, $oldbiblio) = GetBiblio($biblionumber);
684
            ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
757
                $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
758
                ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
759
            } else {
760
                my $authid = $rowref->{'matched_authid'};
761
                ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
762
            }
685
            SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
763
            SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
686
        } elsif ($bib_result eq 'ignore') {
764
        } elsif ($record_result eq 'ignore') {
687
            $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
765
            if ($record_type eq 'biblio') {
766
                $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
767
            }
688
            SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
768
            SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
689
        }
769
        }
690
        # remove matched_biblionumber only if there is no 'imported' item left
770
        my $query;
691
        my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?  AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')" );
771
        if ($record_type eq 'biblio') {
772
            # remove matched_biblionumber only if there is no 'imported' item left
773
            $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
774
            $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?  AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
775
        } else {
776
            $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
777
        }
778
        my $sth2 = $dbh->prepare_cached($query);
692
        $sth2->execute($rowref->{'import_record_id'});
779
        $sth2->execute($rowref->{'import_record_id'});
693
    }
780
    }
694
781
Lines 1263-1268 sub _add_biblio_fields { Link Here
1263
                
1350
                
1264
}
1351
}
1265
1352
1353
sub _add_auth_fields {
1354
    my ($import_record_id, $marc_record) = @_;
1355
1356
    my $dbh = C4::Context->dbh;
1357
    # FIXME no controlnumber
1358
    my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id) VALUES (?)");
1359
    $sth->execute($import_record_id);
1360
    $sth->finish();
1361
                
1362
}
1363
1364
1266
sub _update_biblio_fields {
1365
sub _update_biblio_fields {
1267
    my ($import_record_id, $marc_record) = @_;
1366
    my ($import_record_id, $marc_record) = @_;
1268
1367
Lines 1311-1336 sub _update_batch_record_counts { Link Here
1311
}
1410
}
1312
1411
1313
sub _get_commit_action {
1412
sub _get_commit_action {
1314
    my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_;
1413
    my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1315
    
1414
    
1316
    my ($bib_result, $bib_match, $item_result);
1415
    if ($record_type eq 'biblio') {
1317
1416
        my ($bib_result, $bib_match, $item_result);
1318
    if ($overlay_status ne 'no_match') {
1417
1319
        $bib_match = GetBestRecordMatch($import_record_id);
1418
        if ($overlay_status ne 'no_match') {
1320
        if ($overlay_action eq 'replace') {
1419
            $bib_match = GetBestRecordMatch($import_record_id);
1321
            $bib_result  = defined($bib_match) ? 'replace' : 'create_new';
1420
            if ($overlay_action eq 'replace') {
1322
        } elsif ($overlay_action eq 'create_new') {
1421
                $bib_result  = defined($bib_match) ? 'replace' : 'create_new';
1323
            $bib_result  = 'create_new';
1422
            } elsif ($overlay_action eq 'create_new') {
1324
        } elsif ($overlay_action eq 'ignore') {
1423
                $bib_result  = 'create_new';
1325
            $bib_result  = 'ignore';
1424
            } elsif ($overlay_action eq 'ignore') {
1326
        } 
1425
                $bib_result  = 'ignore';
1327
        $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
1426
            }
1328
    } else {
1427
            $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
1329
        $bib_result = $nomatch_action;
1428
        } else {
1330
        $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new')     ? 'create_new' : 'ignore';
1429
            $bib_result = $nomatch_action;
1331
    }
1430
            $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new')     ? 'create_new' : 'ignore';
1431
        }
1432
        return ($bib_result, $item_result, $bib_match);
1433
    } else { # must be auths
1434
        my ($auth_result, $auth_match);
1435
1436
        if ($overlay_status ne 'no_match') {
1437
            $auth_match = GetBestRecordMatch($import_record_id);
1438
            if ($overlay_action eq 'replace') {
1439
                $auth_result  = defined($auth_match) ? 'replace' : 'create_new';
1440
            } elsif ($overlay_action eq 'create_new') {
1441
                $auth_result  = 'create_new';
1442
            } elsif ($overlay_action eq 'ignore') {
1443
                $auth_result  = 'ignore';
1444
            }
1445
        } else {
1446
            $auth_result = $nomatch_action;
1447
        }
1332
1448
1333
    return ($bib_result, $item_result, $bib_match);
1449
        return ($auth_result, undef, $auth_match);
1450
1451
    }
1334
}
1452
}
1335
1453
1336
sub _get_revert_action {
1454
sub _get_revert_action {
(-)a/C4/Matcher.pm (-14 / +59 lines)
Lines 1-6 Link Here
1
package C4::Matcher;
1
package C4::Matcher;
2
2
3
# Copyright (C) 2007 LibLime
3
# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 22-29 use warnings; Link Here
22
22
23
use C4::Context;
23
use C4::Context;
24
use MARC::Record;
24
use MARC::Record;
25
use C4::Search;
26
use C4::Biblio;
27
25
28
use vars qw($VERSION);
26
use vars qw($VERSION);
29
27
Lines 384-389 sub delete { Link Here
384
    $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
382
    $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
385
}
383
}
386
384
385
=head2 record_type
386
387
  $matcher->record_type('biblio');
388
  my $record_type = $matcher->record_type();
389
390
Accessor method.
391
392
=cut
393
394
sub record_type {
395
    my $self = shift;
396
    @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
397
}
398
387
=head2 threshold
399
=head2 threshold
388
400
389
  $matcher->threshold(1000);
401
  $matcher->threshold(1000);
Lines 582-588 sub add_simple_required_check { Link Here
582
    );
594
    );
583
}
595
}
584
596
585
=head2 find_matches
597
=head2 get_matches
586
598
587
  my @matches = $matcher->get_matches($marc_record, $max_matches);
599
  my @matches = $matcher->get_matches($marc_record, $max_matches);
588
  foreach $match (@matches) {
600
  foreach $match (@matches) {
Lines 618-626 sub get_matches { Link Here
618
        my @source_keys = _get_match_keys($source_record, $matchpoint);
630
        my @source_keys = _get_match_keys($source_record, $matchpoint);
619
        next if scalar(@source_keys) == 0;
631
        next if scalar(@source_keys) == 0;
620
        # build query
632
        # build query
621
        my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
633
        my $query;
622
        # FIXME only searching biblio index at the moment
634
        my $error;
623
        my ($error, $searchresults, $total_hits) = SimpleSearch($query, 0, $max_matches);
635
        my $searchresults;
636
        my $total_hits;
637
        if ($self->{'record_type'} eq 'biblio') {
638
            $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
639
# FIXME only searching biblio index at the moment
640
            require C4::Search;
641
            ($error, $searchresults, $total_hits) = C4::Search::SimpleSearch($query, 0, $max_matches);
642
        } elsif ($self->{'record_type'} eq 'authority') {
643
            my $authresults;
644
            my @marclist;
645
            my @and_or;
646
            my @excluding = [];
647
            my @operator;
648
            my @value;
649
            foreach my $key (@source_keys) {
650
                push @marclist, $matchpoint->{'index'};
651
                push @and_or, 'or';
652
                push @operator, 'exact';
653
                push @value, $key;
654
            }
655
            require C4::AuthoritiesMarc;
656
            ($searchresults, $total_hits) = C4::AuthoritiesMarc::SearchAuthorities(
657
                    \@marclist, \@and_or, \@excluding, \@operator,
658
                    \@value, 0, 20, undef, 'AuthidAsc', 1
659
            );
660
        }
624
661
625
        if (defined $error ) {
662
        if (defined $error ) {
626
            warn "search failed ($query) $error";
663
            warn "search failed ($query) $error";
Lines 639-651 sub get_matches { Link Here
639
                keys %matches;
676
                keys %matches;
640
677
641
    my @results = ();
678
    my @results = ();
642
    foreach my $marcblob (keys %matches) {
679
    if ($self->{'record_type'} eq 'biblio') {
643
        my $target_record = MARC::Record->new_from_usmarc($marcblob);
680
        require C4::Biblio;
644
        my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, '');
681
        foreach my $marcblob (keys %matches) {
645
        # FIXME - again, bibliospecific
682
            my $target_record = MARC::Record->new_from_usmarc($marcblob);
646
        # also, can search engine be induced to give just the number in the first place?
683
            my $record_number;
647
        my $record_number = $result->{'biblionumber'};
684
            my $result = C4::Biblio::TransformMarcToKoha(C4::Context->dbh, $target_record, '');
648
        push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
685
            $record_number = $result->{'biblionumber'};
686
            push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
687
        }
688
    } elsif ($self->{'record_type'} eq 'authority') {
689
        require C4::AuthoritiesMarc;
690
        foreach my $authid (keys %matches) {
691
            push @results, { 'record_id' => $authid, 'score' => $matches{$authid} };
692
        }
649
    }
693
    }
650
    @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
694
    @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
651
    if (scalar(@results) > $max_matches) {
695
    if (scalar(@results) > $max_matches) {
Lines 673-678 sub dump { Link Here
673
    $result->{'matcher_id'} = $self->{'id'};
717
    $result->{'matcher_id'} = $self->{'id'};
674
    $result->{'code'} = $self->{'code'};
718
    $result->{'code'} = $self->{'code'};
675
    $result->{'description'} = $self->{'description'};
719
    $result->{'description'} = $self->{'description'};
720
    $result->{'record_type'} = $self->{'record_type'};
676
721
677
    $result->{'matchpoints'} = [];
722
    $result->{'matchpoints'} = [];
678
    foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
723
    foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
(-)a/admin/matching-rules.pl (-5 / +7 lines)
Lines 92-100 sub add_matching_rule_form { Link Here
92
sub add_update_matching_rule {
92
sub add_update_matching_rule {
93
    my $template = shift;
93
    my $template = shift;
94
    my $matcher_id = shift;
94
    my $matcher_id = shift;
95
    my $record_type = $input->param('record_type') || 'biblio';
95
96
96
    # do parsing
97
    # do parsing
97
    my $matcher = C4::Matcher->new('biblio', 1000); # FIXME biblio only for now
98
    my $matcher = C4::Matcher->new($record_type, 1000);
98
    $matcher->code($input->param('code'));
99
    $matcher->code($input->param('code'));
99
    $matcher->description($input->param('description'));
100
    $matcher->description($input->param('description'));
100
    $matcher->threshold($input->param('threshold'));
101
    $matcher->threshold($input->param('threshold'));
Lines 203-212 sub edit_matching_rule_form { Link Here
203
204
204
    my $matcher = C4::Matcher->fetch($matcher_id);
205
    my $matcher = C4::Matcher->fetch($matcher_id);
205
206
206
    $template->param(matcher_id => $matcher_id);
207
    $template->{VARS}->{'matcher_id'} = $matcher_id;
207
    $template->param(code => $matcher->code());
208
    $template->{VARS}->{'code'} = $matcher->code();
208
    $template->param(description => $matcher->description());
209
    $template->{VARS}->{'description'} = $matcher->description();
209
    $template->param(threshold => $matcher->threshold());
210
    $template->{VARS}->{'threshold'} = $matcher->threshold();
211
    $template->{VARS}->{'record_type'} = $matcher->record_type();
210
212
211
    my $matcher_info = $matcher->dump();
213
    my $matcher_info = $matcher->dump();
212
    my @matchpoints = ();
214
    my @matchpoints = ();
(-)a/installer/data/mysql/atomicupdate/importauthorities.pl (+19 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
use strict;
4
use warnings;
5
use C4::Context;
6
my $dbh = C4::Context->dbh;
7
8
$dbh->do(
9
q|CREATE TABLE `import_auths` (
10
    `import_record_id` int(11) NOT NULL,
11
    `matched_authid` int(11) default NULL,
12
    `control_number` varchar(25) default NULL,
13
    `original_source` varchar(25) default NULL,
14
    CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`)
15
    REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE,
16
    KEY `matched_authid` (`matched_authid`)
17
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;|
18
);
19
print "Upgrade done (Added support for staging authorities)\n";
(-)a/installer/data/mysql/kohastructure.sql (+15 lines)
Lines 909-914 CREATE TABLE `import_record_matches` ( Link Here
909
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
909
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
910
910
911
--
911
--
912
-- Table structure for table `import_auths`
913
--
914
915
DROP TABLE IF EXISTS `import_auths`;
916
CREATE TABLE `import_auths` (
917
  `import_record_id` int(11) NOT NULL,
918
  `matched_authid` int(11) default NULL,
919
  `control_number` varchar(25) default NULL,
920
  `original_source` varchar(25) default NULL,
921
  CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`)
922
             REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE,
923
  KEY `matched_authid` (`matched_authid`),
924
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
925
926
--
912
-- Table structure for table `import_biblios`
927
-- Table structure for table `import_biblios`
913
--
928
--
914
929
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt (+11 lines)
Lines 220-225 function CheckRuleForm(f) { Link Here
220
           <input type="text" id="threshold" name="threshold" size="5" maxlength="5" 
220
           <input type="text" id="threshold" name="threshold" size="5" maxlength="5" 
221
                  value="[% threshold |html %]" /> <span class="required">Required</span>
221
                  value="[% threshold |html %]" /> <span class="required">Required</span>
222
       </li>
222
       </li>
223
       <li><label for="record_type" class="required">Record type: </label>
224
           <select id="record_type" name="record_type">
225
               [% IF ( record_type == "authority" ) %]
226
                   <option value="biblio">Bibliographic record</option>
227
                   <option value="authority" selected="selected">Authority record</option>
228
               [% ELSE %]
229
                   <option value="biblio" selected="selected">Bibliographic record</option>
230
                   <option value="authority">Authority record</option>
231
               [% END %]
232
           </select><span class="required">Required</span>
233
       </li>
223
    </ol>
234
    </ol>
224
  </fieldset>
235
  </fieldset>
225
236
(-)a/misc/commit_biblios_file.pl (-11 / +43 lines)
Lines 18-28 $| = 1; Link Here
18
# command-line parameters
18
# command-line parameters
19
my $batch_number = "";
19
my $batch_number = "";
20
my $list_batches = 0;
20
my $list_batches = 0;
21
my $revert = 0;
21
my $want_help = 0;
22
my $want_help = 0;
22
23
23
my $result = GetOptions(
24
my $result = GetOptions(
24
    'batch-number:s' => \$batch_number,
25
    'batch-number:s' => \$batch_number,
25
    'list-batches'   => \$list_batches,
26
    'list-batches'   => \$list_batches,
27
    'revert'         => \$revert,
26
    'h|help'         => \$want_help
28
    'h|help'         => \$want_help
27
);
29
);
28
30
Lines 45-53 $dbh->{AutoCommit} = 0; Link Here
45
if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
47
if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
46
    my $batch = GetImportBatch($batch_number);
48
    my $batch = GetImportBatch($batch_number);
47
    die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
49
    die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
48
    die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
50
    if ($revert) {
49
        unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
51
        die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
50
    process_batch($batch_number);
52
            unless $batch->{'import_status'} eq "imported";
53
        revert_batch($batch_number);
54
    } else {
55
        die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
56
            unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
57
        process_batch($batch_number);
58
    }
51
    $dbh->commit();
59
    $dbh->commit();
52
} else {
60
} else {
53
    die "$0: please specify a numeric batch ID\n";
61
    die "$0: please specify a numeric batch ID\n";
Lines 74-81 sub process_batch { Link Here
74
    my ($import_batch_id) = @_;
82
    my ($import_batch_id) = @_;
75
83
76
    print "... importing MARC records -- please wait\n";
84
    print "... importing MARC records -- please wait\n";
77
    my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
85
    my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
78
        BatchCommitBibRecords($import_batch_id, '', 100, \&print_progress_and_commit);
86
        BatchCommitRecords($import_batch_id, '', 100, \&print_progress_and_commit);
79
    print "... finished importing MARC records\n";
87
    print "... finished importing MARC records\n";
80
88
81
    print <<_SUMMARY_;
89
    print <<_SUMMARY_;
Lines 83-99 sub process_batch { Link Here
83
MARC record import report
91
MARC record import report
84
----------------------------------------
92
----------------------------------------
85
Batch number:                    $import_batch_id
93
Batch number:                    $import_batch_id
86
Number of new bibs added:        $num_added
94
Number of new records added:     $num_added
87
Number of bibs replaced:         $num_updated
95
Number of records replaced:      $num_updated
88
Number of bibs ignored:          $num_ignored
96
Number of records ignored:       $num_ignored
89
Number of items added:           $num_items_added
97
Number of items added:           $num_items_added
90
Number of items ignored:         $num_items_errored
98
Number of items ignored:         $num_items_errored
91
99
92
Note: an item is ignored if its barcode is a 
100
Note: an item is ignored if its barcode is a
93
duplicate of one already in the database.
101
duplicate of one already in the database.
94
_SUMMARY_
102
_SUMMARY_
95
}
103
}
96
104
105
sub revert_batch {
106
    my ($import_batch_id) = @_;
107
108
    print "... reverting batch -- please wait\n";
109
    my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
110
        BatchRevertRecords($import_batch_id, 100, \&print_progress_and_commit);
111
    print "... finished reverting batch\n";
112
113
    print <<_SUMMARY_;
114
115
MARC record import report
116
----------------------------------------
117
Batch number:                    $import_batch_id
118
Number of records deleted:       $num_deleted
119
Number of errors:                $num_errors
120
Number of records reverted:      $num_reverted
121
Number of records ignored:       $num_ignored
122
Number of items added:           $num_items_deleted
123
124
_SUMMARY_
125
}
126
127
97
sub print_progress_and_commit {
128
sub print_progress_and_commit {
98
    my $recs = shift;
129
    my $recs = shift;
99
    print "... processed $recs records\n";
130
    print "... processed $recs records\n";
Lines 106-112 $0: import a batch of staged MARC records into database. Link Here
106
137
107
Use this batch job to complete the import of a batch of
138
Use this batch job to complete the import of a batch of
108
MARC records that was staged either by the batch job
139
MARC records that was staged either by the batch job
109
stage_biblios_file.pl or by the Koha Tools option
140
stage_file.pl or by the Koha Tools option
110
"Stage MARC Records for Import".
141
"Stage MARC Records for Import".
111
142
112
Parameters:
143
Parameters:
Lines 114-119 Parameters: Link Here
114
                         to import
145
                         to import
115
    --list-batches       print a list of record batches
146
    --list-batches       print a list of record batches
116
                         available to commit
147
                         available to commit
117
    --help or -h            show this message.
148
    --revert             revert a batch instead of importing it
149
    --help or -h         show this message.
118
_USAGE_
150
_USAGE_
119
}
151
}
(-)a/misc/cronjobs/import_webservice_batch.pl (-1 / +1 lines)
Lines 54-57 EOF Link Here
54
my $batch_ids = GetStagedWebserviceBatches() or exit;
54
my $batch_ids = GetStagedWebserviceBatches() or exit;
55
55
56
$framework ||= '';
56
$framework ||= '';
57
BatchCommitBibRecords($_, $framework) foreach @$batch_ids;
57
BatchCommitRecords($_, $framework) foreach @$batch_ids;
(-)a/misc/stage_biblios_file.pl (-36 / +48 lines)
Lines 4-9 Link Here
4
#
4
#
5
# Copyright (C) 2007 LibLime
5
# Copyright (C) 2007 LibLime
6
# Parts Copyright BSZ 2011
6
# Parts Copyright BSZ 2011
7
# Parts Copyright C & P Bibliography Services 2012
7
#
8
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# terms of the GNU General Public License as published by the Free Software
Lines 35-42 use Getopt::Long; Link Here
35
$| = 1;
36
$| = 1;
36
37
37
# command-line parameters
38
# command-line parameters
39
my $record_type = "biblio";
38
my $encoding = "";
40
my $encoding = "";
39
my $match_bibs = 0;
41
my $authorities = 0;
42
my $match = 0;
40
my $add_items = 0;
43
my $add_items = 0;
41
my $input_file = "";
44
my $input_file = "";
42
my $batch_comment = "";
45
my $batch_comment = "";
Lines 46-58 my $no_replace ; Link Here
46
my $result = GetOptions(
49
my $result = GetOptions(
47
    'encoding:s'    => \$encoding,
50
    'encoding:s'    => \$encoding,
48
    'file:s'        => \$input_file,
51
    'file:s'        => \$input_file,
49
    'match-bibs:s'  => \$match_bibs,
52
    'match|match-bibs:s'  => \$match,
50
    'add-items'     => \$add_items,
53
    'add-items'     => \$add_items,
51
    'no-replace'    => \$no_replace,
54
    'no-replace'    => \$no_replace,
52
    'comment:s'     => \$batch_comment,
55
    'comment:s'     => \$batch_comment,
56
    'authorities'   => \$authorities,
53
    'h|help'        => \$want_help
57
    'h|help'        => \$want_help
54
);
58
);
55
59
60
$record_type = 'auth' if ($authorities);
61
56
if ($encoding eq "") {
62
if ($encoding eq "") {
57
    $encoding = "utf8";
63
    $encoding = "utf8";
58
}
64
}
Lines 68-80 unless (-r $input_file) { Link Here
68
74
69
my $dbh = C4::Context->dbh;
75
my $dbh = C4::Context->dbh;
70
$dbh->{AutoCommit} = 0;
76
$dbh->{AutoCommit} = 0;
71
process_batch($input_file, $match_bibs, $add_items, $batch_comment);
77
process_batch($input_file, $record_type, $match, $add_items, $batch_comment);
72
$dbh->commit();
78
$dbh->commit();
73
79
74
exit 0;
80
exit 0;
75
81
76
sub process_batch {
82
sub process_batch {
77
    my ($input_file, $match_bibs, $add_items, $batch_comment) = @_;
83
    my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_;
78
84
79
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
85
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
80
    my $marc_records = "";
86
    my $marc_records = "";
Lines 92-141 sub process_batch { Link Here
92
    close IN;
98
    close IN;
93
99
94
    print "... staging MARC records -- please wait\n";
100
    print "... staging MARC records -- please wait\n";
95
    my ($batch_id, $num_valid, $num_items, @import_errors) = 
101
    my ($batch_id, $num_valid_records, $num_items, @import_errors) =
96
        BatchStageMarcRecords($encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
102
        BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
97
                              100, \&print_progress_and_commit);
103
                              100, \&print_progress_and_commit);
98
    print "... finished staging MARC records\n";
104
    print "... finished staging MARC records\n";
99
105
100
    my $num_with_matches = 0;
106
    my $num_with_matches = 0;
101
    if ($match_bibs) {
107
    if ($match) {
102
        my $matcher = C4::Matcher->fetch($match_bibs) ;
108
        my $matcher = C4::Matcher->fetch($match) ;
103
        if (! defined $matcher) {
109
        if (defined $matcher) {
104
            $matcher = C4::Matcher->new('biblio');
110
            SetImportBatchMatcher($batch_id, $match);
111
        } elsif ($record_type eq 'biblio')  {
112
            $matcher = C4::Matcher->new($record_type);
105
            $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
113
            $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
106
            $matcher->add_simple_required_check('245', 'a', -1, 0, '', 
114
            $matcher->add_simple_required_check('245', 'a', -1, 0, '',
107
                                            '245', 'a', -1, 0, '');
115
                                            '245', 'a', -1, 0, '');
108
        } else {
109
            SetImportBatchMatcher($batch_id, $match_bibs);
110
        }
116
        }
111
        # set default record overlay behavior
117
        # set default record overlay behavior
112
        SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace');
118
        SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace');
113
        SetImportBatchNoMatchAction($batch_id, 'create_new');
119
        SetImportBatchNoMatchAction($batch_id, 'create_new');
114
        SetImportBatchItemAction($batch_id, 'always_add');
120
        SetImportBatchItemAction($batch_id, 'always_add');
115
        print "... looking for matches with records already in database\n";
121
        print "... looking for matches with records already in database\n";
116
        $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
122
        $num_with_matches = BatchFindDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
117
        print "... finished looking for matches\n";
123
        print "... finished looking for matches\n";
118
    }
124
    }
119
125
120
    my $num_invalid_bibs = scalar(@import_errors);
126
    my $num_invalid_records = scalar(@import_errors);
121
    print <<_SUMMARY_;
127
    print <<_SUMMARY_;
122
128
123
MARC record staging report
129
MARC record staging report
124
------------------------------------
130
------------------------------------
125
Input file:              $input_file
131
Input file:                 $input_file
126
Number of input bibs:    $num_input_records
132
Record type:                $record_type
127
Number of valid bibs:    $num_valid
133
Number of input records:    $num_input_records
128
Number of invalid bibs:  $num_invalid_bibs
134
Number of valid records:    $num_valid_records
135
Number of invalid records:  $num_invalid_records
129
_SUMMARY_
136
_SUMMARY_
130
    if ($match_bibs) {
137
    if ($match) {
131
        print "Number of bibs matched:  $num_with_matches\n";
138
        print "Number of records matched:  $num_with_matches\n";
132
    } else {
139
    } else {
133
        print "Incoming bibs not matched against existing bibs (--match-bibs option not supplied)\n";
140
        print "Incoming records not matched against existing records (--match option not supplied)\n";
134
    }
141
    }
135
    if ($add_items) {
142
    if ($record_type eq 'biblio') {
136
        print "Number of items parsed:  $num_items\n";
143
        if ($add_items) {
137
    } else {
144
            print "Number of items parsed:  $num_items\n";
138
        print "No items parsed (--add-items option not supplied)\n";
145
        } else {
146
            print "No items parsed (--add-items option not supplied)\n";
147
        }
139
    }
148
    }
140
149
141
    print "\n";
150
    print "\n";
Lines 151-181 sub print_progress_and_commit { Link Here
151
160
152
sub print_usage {
161
sub print_usage {
153
    print <<_USAGE_;
162
    print <<_USAGE_;
154
$0: stage MARC bib file into reservoir.
163
$0: stage MARC file into reservoir.
155
164
156
Use this batch job to load a file of MARC bibliographic records
165
Use this batch job to load a file of MARC bibliographic
157
(with optional item information) into the Koha reservoir.
166
(with optional item information) or authority records into
167
the Koha reservoir.
158
168
159
After running this program to stage your file, you can use
169
After running this program to stage your file, you can use
160
either the batch job commit_biblios_file.pl or the Koha
170
either the batch job commit_file.pl or the Koha
161
Tools option "Manage Staged MARC Records" to load the
171
Tools option "Manage Staged MARC Records" to load the
162
records into the main Koha database.
172
records into the main Koha database.
163
173
164
Parameters:
174
Parameters:
165
    --file <file_name>      name of input MARC bib file
175
    --file <file_name>      name of input MARC bib file
176
    --authorities           stage authority records instead of bibs
166
    --encoding <encoding>   encoding of MARC records, default is utf8.
177
    --encoding <encoding>   encoding of MARC records, default is utf8.
167
                            Other possible options are: MARC-8,
178
                            Other possible options are: MARC-8,
168
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
179
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
169
    --match-bibs <match_id> use this option to match bibs
180
    --match <match_id>      use this option to match records
170
                            in the file with bibs already in 
181
                            in the file with records already in
171
                            the database for future overlay.
182
                            the database for future overlay.
172
                            If <match_id> isn't defined, a default 
183
                            If <match_id> isn't defined, a default
173
                            MARC21 ISBN & title match rule will be applied.
184
                            MARC21 ISBN & title match rule will be applied
185
                            for bib imports.
174
    --add-items             use this option to specify that
186
    --add-items             use this option to specify that
175
                            item data is embedded in the MARC
187
                            item data is embedded in the MARC
176
                            bibs and should be parsed.
188
                            bibs and should be parsed.
177
    --no-replace            overlay action for bib record: default is to 
189
    --no-replace            overlay action for record: default is to
178
                            replace extant bib with the imported record.
190
                            replace extant with the imported record.
179
    --comment <comment>     optional comment to describe
191
    --comment <comment>     optional comment to describe
180
                            the record batch; if the comment
192
                            the record batch; if the comment
181
                            has spaces in it, surround the
193
                            has spaces in it, surround the
(-)a/svc/import_bib (-1 / +1 lines)
Lines 92-98 sub import_bib { Link Here
92
    $marcxml =~ s/<\?xml.*?\?>//i;
92
    $marcxml =~ s/<\?xml.*?\?>//i;
93
93
94
    # XXX we are ignoring the result of this;
94
    # XXX we are ignoring the result of this;
95
    BatchCommitBibRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
95
    BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
96
96
97
    $result->{'status'} = "ok";
97
    $result->{'status'} = "ok";
98
    $result->{'import_batch_id'} =  $batch_id;
98
    $result->{'import_batch_id'} =  $batch_id;
(-)a/tools/manage-marc-import.pl (-3 / +2 lines)
Lines 244-250 sub commit_batch { Link Here
244
        $callback = progress_callback($job, $dbh);
244
        $callback = progress_callback($job, $dbh);
245
    }
245
    }
246
    my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
246
    my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
247
        BatchCommitBibRecords($import_batch_id, $framework, 50, $callback);
247
        BatchCommitRecords($import_batch_id, $framework, 50, $callback);
248
    $dbh->commit();
248
    $dbh->commit();
249
249
250
    my $results = {
250
    my $results = {
Lines 273-279 sub revert_batch { Link Here
273
        $callback = progress_callback($job, $dbh);
273
        $callback = progress_callback($job, $dbh);
274
    }
274
    }
275
    my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = 
275
    my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = 
276
        BatchRevertBibRecords($import_batch_id, 50, $callback);
276
        BatchRevertRecords($import_batch_id, 50, $callback);
277
    $dbh->commit();
277
    $dbh->commit();
278
278
279
    my $results = {
279
    my $results = {
280
- 

Return to bug 7475