View | Details | Raw Unified | Return to bug 16506
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-56 / +60 lines)
Lines 1-7 Link Here
1
#!/usr/bin/perl
1
#!/usr/bin/perl
2
2
3
use strict;
3
# This file is part of Koha.
4
#use warnings; FIXME - Bug 2505
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
5
19
6
use C4::Context;
20
use C4::Context;
7
use Getopt::Long;
21
use Getopt::Long;
Lines 34-43 my $skip_index; Link Here
34
my $reset;
48
my $reset;
35
my $biblios;
49
my $biblios;
36
my $authorities;
50
my $authorities;
37
my $noxml;
51
my $as_usmarc;
52
my $as_xml;
38
my $noshadow;
53
my $noshadow;
39
my $want_help;
54
my $want_help;
40
my $as_xml;
41
my $process_zebraqueue;
55
my $process_zebraqueue;
42
my $process_zebraqueue_skip_deletes;
56
my $process_zebraqueue_skip_deletes;
43
my $do_not_clear_zebraqueue;
57
my $do_not_clear_zebraqueue;
Lines 62-68 my $result = GetOptions( Link Here
62
    'I|skip-index'  => \$skip_index,
76
    'I|skip-index'  => \$skip_index,
63
    'nosanitize'    => \$nosanitize,
77
    'nosanitize'    => \$nosanitize,
64
    'b'             => \$biblios,
78
    'b'             => \$biblios,
65
    'noxml'         => \$noxml,
79
    'noxml'         => \$as_usmarc,
66
    'w'             => \$noshadow,
80
    'w'             => \$noshadow,
67
    'a'             => \$authorities,
81
    'a'             => \$authorities,
68
    'h|help'        => \$want_help,
82
    'h|help'        => \$want_help,
Lines 84-89 if (not $result or $want_help) { Link Here
84
    exit 0;
98
    exit 0;
85
}
99
}
86
100
101
if ( $as_xml ) {
102
    warn "Warning: You passed -x which is already the default and is now deprecated·\n";
103
}
104
87
if( not defined $run_as_root and $run_user eq 'root') {
105
if( not defined $run_as_root and $run_user eq 'root') {
88
    my $msg = "Warning: You are running this script as the user 'root'.\n";
106
    my $msg = "Warning: You are running this script as the user 'root'.\n";
89
    $msg   .= "If this is intentional you must explicitly specify this using the -run-as-root switch\n";
107
    $msg   .= "If this is intentional you must explicitly specify this using the -run-as-root switch\n";
Lines 91-97 if( not defined $run_as_root and $run_user eq 'root') { Link Here
91
    die $msg;
109
    die $msg;
92
}
110
}
93
111
94
if ( !$as_xml and $nosanitize ) {
112
if ( $as_usmarc and $nosanitize ) {
95
    my $msg = "Cannot specify both -no_xml and -nosanitize\n";
113
    my $msg = "Cannot specify both -no_xml and -nosanitize\n";
96
    $msg   .= "Please do '$0 --help' to see usage.\n";
114
    $msg   .= "Please do '$0 --help' to see usage.\n";
97
    die $msg;
115
    die $msg;
Lines 277-289 if ($keep_export) { Link Here
277
295
278
sub do_one_pass {
296
sub do_one_pass {
279
    if ($authorities) {
297
    if ($authorities) {
280
        index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
298
        index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
281
    } else {
299
    } else {
282
        print "skipping authorities\n" if ( $verbose_logging );
300
        print "skipping authorities\n" if ( $verbose_logging );
283
    }
301
    }
284
302
285
    if ($biblios) {
303
    if ($biblios) {
286
        index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
304
        index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
287
    } else {
305
    } else {
288
        print "skipping biblios\n" if ( $verbose_logging );
306
        print "skipping biblios\n" if ( $verbose_logging );
289
    }
307
    }
Lines 330-336 sub check_zebra_dirs { Link Here
330
}   # ----------  end of subroutine check_zebra_dirs  ----------
348
}   # ----------  end of subroutine check_zebra_dirs  ----------
331
349
332
sub index_records {
350
sub index_records {
333
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
351
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
334
352
335
    my $num_records_exported = 0;
353
    my $num_records_exported = 0;
336
    my $records_deleted = {};
354
    my $records_deleted = {};
Lines 357-374 sub index_records { Link Here
357
            unless ( $process_zebraqueue_skip_deletes ) {
375
            unless ( $process_zebraqueue_skip_deletes ) {
358
                $entries = select_zebraqueue_records($record_type, 'deleted');
376
                $entries = select_zebraqueue_records($record_type, 'deleted');
359
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
377
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
360
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
378
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_usmarc);
361
                mark_zebraqueue_batch_done($entries);
379
                mark_zebraqueue_batch_done($entries);
362
            }
380
            }
363
381
364
            $entries = select_zebraqueue_records($record_type, 'updated');
382
            $entries = select_zebraqueue_records($record_type, 'updated');
365
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
383
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
366
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
384
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_usmarc, $records_deleted);
367
            mark_zebraqueue_batch_done($entries);
385
            mark_zebraqueue_batch_done($entries);
368
386
369
        } else {
387
        } else {
370
            my $sth = select_all_records($record_type);
388
            my $sth = select_all_records($record_type);
371
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
389
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_usmarc, $nosanitize);
372
            unless ($do_not_clear_zebraqueue) {
390
            unless ($do_not_clear_zebraqueue) {
373
                mark_all_zebraqueue_done($record_type);
391
                mark_all_zebraqueue_done($record_type);
374
            }
392
            }
Lines 390-396 sub index_records { Link Here
390
            print "REINDEXING zebra\n";
408
            print "REINDEXING zebra\n";
391
            print "====================\n";
409
            print "====================\n";
392
        }
410
        }
393
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
411
        my $record_fmt = ($as_usmarc) ? 'iso2709' : 'marcxml' ;
394
        if ($process_zebraqueue) {
412
        if ($process_zebraqueue) {
395
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
413
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
396
                if %$records_deleted;
414
                if %$records_deleted;
Lines 470-494 sub select_all_biblios { Link Here
470
    return $sth;
488
    return $sth;
471
}
489
}
472
490
473
sub include_xml_wrapper {
474
    my $as_xml = shift;
475
    my $record_type = shift;
476
477
    return 0 unless $as_xml;
478
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
479
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
480
    return 0;
481
482
}
483
484
sub export_marc_records_from_sth {
491
sub export_marc_records_from_sth {
485
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
492
    my ($record_type, $sth, $directory, $as_usmarc, $nosanitize) = @_;
486
493
487
    my $num_exported = 0;
494
    my $num_exported = 0;
488
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
495
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
489
496
490
    print {$fh} $marcxml_open
497
    print {$fh} $marcxml_open
491
        if include_xml_wrapper($as_xml, $record_type);
498
        unless $as_usmarc;
492
499
493
    my $i = 0;
500
    my $i = 0;
494
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
501
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
Lines 532-542 sub export_marc_records_from_sth { Link Here
532
            }
539
            }
533
            next;
540
            next;
534
        }
541
        }
535
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
542
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc);
536
        if (defined $marc) {
543
        if (defined $marc) {
537
            eval {
544
            eval {
538
                my $rec;
545
                my $rec;
539
                if ($as_xml) {
546
                if ($as_usmarc) {
547
                    $rec = $marc->as_usmarc();
548
                } else {
540
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
549
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
541
                    eval {
550
                    eval {
542
                        my $doc = $tester->parse_string($rec);
551
                        my $doc = $tester->parse_string($rec);
Lines 545-577 sub export_marc_records_from_sth { Link Here
545
                        die "invalid XML: $@";
554
                        die "invalid XML: $@";
546
                    }
555
                    }
547
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
556
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
548
                } else {
549
                    $rec = $marc->as_usmarc();
550
                }
557
                }
551
                print {$fh} $rec;
558
                print {$fh} $rec;
552
                $num_exported++;
559
                $num_exported++;
553
            };
560
            };
554
            if ($@) {
561
            if ($@) {
555
                warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
562
                warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML");
556
                warn "... specific error is $@" if $verbose_logging;
563
                warn "... specific error is $@" if $verbose_logging;
557
            }
564
            }
558
        }
565
        }
559
    }
566
    }
560
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
567
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
561
    print {$fh} $marcxml_close
568
    print {$fh} $marcxml_close
562
        if include_xml_wrapper($as_xml, $record_type);
569
        unless $as_usmarc;
570
563
    close $fh;
571
    close $fh;
564
    return $num_exported;
572
    return $num_exported;
565
}
573
}
566
574
567
sub export_marc_records_from_list {
575
sub export_marc_records_from_list {
568
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
576
    my ($record_type, $entries, $directory, $as_usmarc, $records_deleted) = @_;
569
577
570
    my $num_exported = 0;
578
    my $num_exported = 0;
571
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
579
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
572
580
573
    print {$fh} $marcxml_open
581
    print {$fh} $marcxml_open
574
        if include_xml_wrapper($as_xml, $record_type);
582
        unless $as_usmarc;
575
583
576
    my $i = 0;
584
    my $i = 0;
577
585
Lines 582-622 sub export_marc_records_from_list { Link Here
582
                                @$entries ) {
590
                                @$entries ) {
583
        print "." if ( $verbose_logging );
591
        print "." if ( $verbose_logging );
584
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
592
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
585
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
593
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc);
586
        if (defined $marc) {
594
        if (defined $marc) {
587
            eval {
595
            eval {
588
                my $rec;
596
                my $rec;
589
                if ($as_xml) {
597
                if ( $as_usmarc ) {
598
                    $rec = $marc->as_usmarc();
599
                } else {
590
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
600
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
591
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
601
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
592
                } else {
593
                    $rec = $marc->as_usmarc();
594
                }
602
                }
595
                print {$fh} $rec;
603
                print {$fh} $rec;
596
                $num_exported++;
604
                $num_exported++;
597
            };
605
            };
598
            if ($@) {
606
            if ($@) {
599
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
607
              warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML");
600
            }
608
            }
601
        }
609
        }
602
    }
610
    }
603
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
611
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
604
612
605
    print {$fh} $marcxml_close
613
    print {$fh} $marcxml_close
606
        if include_xml_wrapper($as_xml, $record_type);
614
        unless $as_usmarc;
607
615
608
    close $fh;
616
    close $fh;
609
    return $num_exported;
617
    return $num_exported;
610
}
618
}
611
619
612
sub generate_deleted_marc_records {
620
sub generate_deleted_marc_records {
613
    my ($record_type, $entries, $directory, $as_xml) = @_;
621
622
    my ($record_type, $entries, $directory, $as_usmarc) = @_;
614
623
615
    my $records_deleted = {};
624
    my $records_deleted = {};
616
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
625
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
617
626
618
    print {$fh} $marcxml_open
627
    print {$fh} $marcxml_open
619
        if include_xml_wrapper($as_xml, $record_type);
628
        unless $as_usmarc;
620
629
621
    my $i = 0;
630
    my $i = 0;
622
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
631
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
Lines 634-644 sub generate_deleted_marc_records { Link Here
634
        }
643
        }
635
644
636
        my $rec;
645
        my $rec;
637
        if ($as_xml) {
646
        if ( $as_usmarc ) {
647
            $rec = $marc->as_usmarc();
648
        } else {
638
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
649
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
650
            # Remove the record's XML header
639
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
651
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
640
        } else {
641
            $rec = $marc->as_usmarc();
642
        }
652
        }
643
        print {$fh} $rec;
653
        print {$fh} $rec;
644
654
Lines 647-664 sub generate_deleted_marc_records { Link Here
647
    print "\nRecords exported: $i\n" if ( $verbose_logging );
657
    print "\nRecords exported: $i\n" if ( $verbose_logging );
648
658
649
    print {$fh} $marcxml_close
659
    print {$fh} $marcxml_close
650
        if include_xml_wrapper($as_xml, $record_type);
660
        unless $as_usmarc;
651
661
652
    close $fh;
662
    close $fh;
653
    return $records_deleted;
663
    return $records_deleted;
654
655
656
}
664
}
657
665
658
sub get_corrected_marc_record {
666
sub get_corrected_marc_record {
659
    my ($record_type, $record_number, $noxml) = @_;
667
    my ($record_type, $record_number, $as_usmarc) = @_;
660
668
661
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
669
    my $marc = get_raw_marc_record($record_type, $record_number, $as_usmarc);
662
670
663
    if (defined $marc) {
671
    if (defined $marc) {
664
        fix_leader($marc);
672
        fix_leader($marc);
Lines 677-687 sub get_corrected_marc_record { Link Here
677
}
685
}
678
686
679
sub get_raw_marc_record {
687
sub get_raw_marc_record {
680
    my ($record_type, $record_number, $noxml) = @_;
688
    my ($record_type, $record_number, $as_usmarc) = @_;
681
689
682
    my $marc;
690
    my $marc;
683
    if ($record_type eq 'biblio') {
691
    if ($record_type eq 'biblio') {
684
        if ($noxml) {
692
        if ($as_usmarc) {
685
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
693
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
686
            $fetch_sth->execute($record_number);
694
            $fetch_sth->execute($record_number);
687
            if (my ($blob) = $fetch_sth->fetchrow_array) {
695
            if (my ($blob) = $fetch_sth->fetchrow_array) {
Lines 908-916 Parameters: Link Here
908
                            option is recommended only
916
                            option is recommended only
909
                            for advanced user.
917
                            for advanced user.
910
918
911
    -x                      export and index as xml instead of is02709 (biblios only).
912
                            use this if you might have records > 99,999 chars,
913
914
    -nosanitize             export biblio/authority records directly from DB marcxml
919
    -nosanitize             export biblio/authority records directly from DB marcxml
915
                            field without sanitizing records. It speed up
920
                            field without sanitizing records. It speed up
916
                            dump process but could fail if DB contains badly
921
                            dump process but could fail if DB contains badly
917
- 

Return to bug 16506