View | Details | Raw Unified | Return to bug 17731
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-94 / +42 lines)
Lines 48-54 my $skip_index; Link Here
48
my $reset;
48
my $reset;
49
my $biblios;
49
my $biblios;
50
my $authorities;
50
my $authorities;
51
my $as_usmarc;
52
my $as_xml;
51
my $as_xml;
53
my $noshadow;
52
my $noshadow;
54
my $want_help;
53
my $want_help;
Lines 76-82 my $result = GetOptions( Link Here
76
    'I|skip-index'  => \$skip_index,
75
    'I|skip-index'  => \$skip_index,
77
    'nosanitize'    => \$nosanitize,
76
    'nosanitize'    => \$nosanitize,
78
    'b'             => \$biblios,
77
    'b'             => \$biblios,
79
    'noxml'         => \$as_usmarc,
80
    'w'             => \$noshadow,
78
    'w'             => \$noshadow,
81
    'a'             => \$authorities,
79
    'a'             => \$authorities,
82
    'h|help'        => \$want_help,
80
    'h|help'        => \$want_help,
Lines 110-121 if( not defined $run_as_root and $run_user eq 'root') { Link Here
110
    die $msg;
108
    die $msg;
111
}
109
}
112
110
113
if ( $as_usmarc and $nosanitize ) {
114
    my $msg = "Cannot specify both -noxml and -nosanitize\n";
115
    $msg   .= "Please do '$0 --help' to see usage.\n";
116
    die $msg;
117
}
118
119
if ($process_zebraqueue and ($skip_export or $reset)) {
111
if ($process_zebraqueue and ($skip_export or $reset)) {
120
    my $msg = "Cannot specify -r or -s if -z is specified\n";
112
    my $msg = "Cannot specify -r or -s if -z is specified\n";
121
    $msg   .= "Please do '$0 --help' to see usage.\n";
113
    $msg   .= "Please do '$0 --help' to see usage.\n";
Lines 295-307 if ($keep_export) { Link Here
295
287
296
sub do_one_pass {
288
sub do_one_pass {
297
    if ($authorities) {
289
    if ($authorities) {
298
        index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
290
        index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
299
    } else {
291
    } else {
300
        print "skipping authorities\n" if ( $verbose_logging );
292
        print "skipping authorities\n" if ( $verbose_logging );
301
    }
293
    }
302
294
303
    if ($biblios) {
295
    if ($biblios) {
304
        index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
296
        index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
305
    } else {
297
    } else {
306
        print "skipping biblios\n" if ( $verbose_logging );
298
        print "skipping biblios\n" if ( $verbose_logging );
307
    }
299
    }
Lines 348-354 sub check_zebra_dirs { Link Here
348
}   # ----------  end of subroutine check_zebra_dirs  ----------
340
}   # ----------  end of subroutine check_zebra_dirs  ----------
349
341
350
sub index_records {
342
sub index_records {
351
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
343
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
352
344
353
    my $num_records_exported = 0;
345
    my $num_records_exported = 0;
354
    my $records_deleted = {};
346
    my $records_deleted = {};
Lines 375-392 sub index_records { Link Here
375
            unless ( $process_zebraqueue_skip_deletes ) {
367
            unless ( $process_zebraqueue_skip_deletes ) {
376
                $entries = select_zebraqueue_records($record_type, 'deleted');
368
                $entries = select_zebraqueue_records($record_type, 'deleted');
377
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
369
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
378
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_usmarc);
370
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type");
379
                mark_zebraqueue_batch_done($entries);
371
                mark_zebraqueue_batch_done($entries);
380
            }
372
            }
381
373
382
            $entries = select_zebraqueue_records($record_type, 'updated');
374
            $entries = select_zebraqueue_records($record_type, 'updated');
383
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
375
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
384
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_usmarc, $records_deleted);
376
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $records_deleted);
385
            mark_zebraqueue_batch_done($entries);
377
            mark_zebraqueue_batch_done($entries);
386
378
387
        } else {
379
        } else {
388
            my $sth = select_all_records($record_type);
380
            my $sth = select_all_records($record_type);
389
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_usmarc, $nosanitize);
381
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize);
390
            unless ($do_not_clear_zebraqueue) {
382
            unless ($do_not_clear_zebraqueue) {
391
                mark_all_zebraqueue_done($record_type);
383
                mark_all_zebraqueue_done($record_type);
392
            }
384
            }
Lines 408-414 sub index_records { Link Here
408
            print "REINDEXING zebra\n";
400
            print "REINDEXING zebra\n";
409
            print "====================\n";
401
            print "====================\n";
410
        }
402
        }
411
        my $record_fmt = ($as_usmarc) ? 'iso2709' : 'marcxml' ;
403
        my $record_fmt = 'marcxml';
412
        if ($process_zebraqueue) {
404
        if ($process_zebraqueue) {
413
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
405
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
414
                if %$records_deleted;
406
                if %$records_deleted;
Lines 489-501 sub select_all_biblios { Link Here
489
}
481
}
490
482
491
sub export_marc_records_from_sth {
483
sub export_marc_records_from_sth {
492
    my ($record_type, $sth, $directory, $as_usmarc, $nosanitize) = @_;
484
    my ($record_type, $sth, $directory, $nosanitize) = @_;
493
485
494
    my $num_exported = 0;
486
    my $num_exported = 0;
495
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
487
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
496
488
497
    print {$fh} $marcxml_open
489
    print {$fh} $marcxml_open;
498
        unless $as_usmarc;
499
490
500
    my $i = 0;
491
    my $i = 0;
501
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
492
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
Lines 539-585 sub export_marc_records_from_sth { Link Here
539
            }
530
            }
540
            next;
531
            next;
541
        }
532
        }
542
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc);
533
        my ($marc) = get_corrected_marc_record($record_type, $record_number);
543
        if (defined $marc) {
534
        if (defined $marc) {
544
            eval {
535
            eval {
545
                my $rec;
536
                my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
546
                if ($as_usmarc) {
537
                eval {
547
                    $rec = $marc->as_usmarc();
538
                    my $doc = $tester->parse_string($rec);
548
                } else {
539
                };
549
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
540
                if ($@) {
550
                    eval {
541
                    die "invalid XML: $@";
551
                        my $doc = $tester->parse_string($rec);
552
                    };
553
                    if ($@) {
554
                        die "invalid XML: $@";
555
                    }
556
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
557
                }
542
                }
543
                $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
558
                print {$fh} $rec;
544
                print {$fh} $rec;
559
                $num_exported++;
545
                $num_exported++;
560
            };
546
            };
561
            if ($@) {
547
            if ($@) {
562
                warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML");
548
                warn "Error exporting record $record_number ($record_type) XML";
563
                warn "... specific error is $@" if $verbose_logging;
549
                warn "... specific error is $@" if $verbose_logging;
564
            }
550
            }
565
        }
551
        }
566
    }
552
    }
567
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
553
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
568
    print {$fh} $marcxml_close
554
    print {$fh} $marcxml_close;
569
        unless $as_usmarc;
570
555
571
    close $fh;
556
    close $fh;
572
    return $num_exported;
557
    return $num_exported;
573
}
558
}
574
559
575
sub export_marc_records_from_list {
560
sub export_marc_records_from_list {
576
    my ($record_type, $entries, $directory, $as_usmarc, $records_deleted) = @_;
561
    my ($record_type, $entries, $directory, $records_deleted) = @_;
577
562
578
    my $num_exported = 0;
563
    my $num_exported = 0;
579
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
564
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
580
565
581
    print {$fh} $marcxml_open
566
    print {$fh} $marcxml_open;
582
        unless $as_usmarc;
583
567
584
    my $i = 0;
568
    my $i = 0;
585
569
Lines 590-617 sub export_marc_records_from_list { Link Here
590
                                @$entries ) {
574
                                @$entries ) {
591
        print "." if ( $verbose_logging );
575
        print "." if ( $verbose_logging );
592
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
576
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
593
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc);
577
        my ($marc) = get_corrected_marc_record($record_type, $record_number);
594
        if (defined $marc) {
578
        if (defined $marc) {
595
            eval {
579
            eval {
596
                my $rec;
580
                my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
597
                if ( $as_usmarc ) {
581
                $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
598
                    $rec = $marc->as_usmarc();
599
                } else {
600
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
601
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
602
                }
603
                print {$fh} $rec;
582
                print {$fh} $rec;
604
                $num_exported++;
583
                $num_exported++;
605
            };
584
            };
606
            if ($@) {
585
            if ($@) {
607
              warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML");
586
              warn "Error exporting record $record_number ($record_type) XML";
608
            }
587
            }
609
        }
588
        }
610
    }
589
    }
611
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
590
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
612
591
613
    print {$fh} $marcxml_close
592
    print {$fh} $marcxml_close;
614
        unless $as_usmarc;
615
593
616
    close $fh;
594
    close $fh;
617
    return $num_exported;
595
    return $num_exported;
Lines 619-631 sub export_marc_records_from_list { Link Here
619
597
620
sub generate_deleted_marc_records {
598
sub generate_deleted_marc_records {
621
599
622
    my ($record_type, $entries, $directory, $as_usmarc) = @_;
600
    my ($record_type, $entries, $directory) = @_;
623
601
624
    my $records_deleted = {};
602
    my $records_deleted = {};
625
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
603
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
626
604
627
    print {$fh} $marcxml_open
605
    print {$fh} $marcxml_open;
628
        unless $as_usmarc;
629
606
630
    my $i = 0;
607
    my $i = 0;
631
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
608
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
Lines 642-672 sub generate_deleted_marc_records { Link Here
642
            fix_unimarc_100($marc);
619
            fix_unimarc_100($marc);
643
        }
620
        }
644
621
645
        my $rec;
622
        my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
646
        if ( $as_usmarc ) {
623
        # Remove the record's XML header
647
            $rec = $marc->as_usmarc();
624
        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
648
        } else {
649
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
650
            # Remove the record's XML header
651
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
652
        }
653
        print {$fh} $rec;
625
        print {$fh} $rec;
654
626
655
        $records_deleted->{$record_number} = 1;
627
        $records_deleted->{$record_number} = 1;
656
    }
628
    }
657
    print "\nRecords exported: $i\n" if ( $verbose_logging );
629
    print "\nRecords exported: $i\n" if ( $verbose_logging );
658
630
659
    print {$fh} $marcxml_close
631
    print {$fh} $marcxml_close;
660
        unless $as_usmarc;
661
632
662
    close $fh;
633
    close $fh;
663
    return $records_deleted;
634
    return $records_deleted;
664
}
635
}
665
636
666
sub get_corrected_marc_record {
637
sub get_corrected_marc_record {
667
    my ($record_type, $record_number, $as_usmarc) = @_;
638
    my ($record_type, $record_number) = @_;
668
639
669
    my $marc = get_raw_marc_record($record_type, $record_number, $as_usmarc);
640
    my $marc = get_raw_marc_record($record_type, $record_number);
670
641
671
    if (defined $marc) {
642
    if (defined $marc) {
672
        fix_leader($marc);
643
        fix_leader($marc);
Lines 685-718 sub get_corrected_marc_record { Link Here
685
}
656
}
686
657
687
sub get_raw_marc_record {
658
sub get_raw_marc_record {
688
    my ($record_type, $record_number, $as_usmarc) = @_;
659
    my ($record_type, $record_number) = @_;
689
660
690
    my $marc;
661
    my $marc;
691
    if ($record_type eq 'biblio') {
662
    if ($record_type eq 'biblio') {
692
        if ($as_usmarc) {
663
        eval { $marc = GetMarcBiblio($record_number, 1); };
693
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
664
        if ($@ || !$marc) {
694
            $fetch_sth->execute($record_number);
665
            # here we do warn since catching an exception
695
            if (my ($blob) = $fetch_sth->fetchrow_array) {
666
            # means that the bib was found but failed
696
                $marc = MARC::Record->new_from_usmarc($blob);
667
            # to be parsed
697
                unless ($marc) {
668
            warn "error retrieving biblio $record_number";
698
                    warn "error creating MARC::Record from $blob";
669
            return;
699
                }
700
            }
701
            # failure to find a bib is not a problem -
702
            # a delete could have been done before
703
            # trying to process a record update
704
705
            $fetch_sth->finish();
706
            return unless $marc;
707
        } else {
708
            eval { $marc = GetMarcBiblio($record_number, 1); };
709
            if ($@ || !$marc) {
710
                # here we do warn since catching an exception
711
                # means that the bib was found but failed
712
                # to be parsed
713
                warn "error retrieving biblio $record_number";
714
                return;
715
            }
716
        }
670
        }
717
    } else {
671
    } else {
718
        eval { $marc = GetAuthority($record_number); };
672
        eval { $marc = GetAuthority($record_number); };
Lines 918-928 Parameters: Link Here
918
                            already exported the records
872
                            already exported the records
919
                            in a previous run.
873
                            in a previous run.
920
874
921
    -noxml                  index from ISO MARC blob
922
                            instead of MARC XML.  This
923
                            option is recommended only
924
                            for advanced user.
925
926
    -nosanitize             export biblio/authority records directly from DB marcxml
875
    -nosanitize             export biblio/authority records directly from DB marcxml
927
                            field without sanitizing records. It speed up
876
                            field without sanitizing records. It speed up
928
                            dump process but could fail if DB contains badly
877
                            dump process but could fail if DB contains badly
929
- 

Return to bug 17731