Lines 1-7
Link Here
|
1 |
#!/usr/bin/perl |
1 |
#!/usr/bin/perl |
2 |
|
2 |
|
3 |
use strict; |
3 |
# This file is part of Koha. |
4 |
#use warnings; FIXME - Bug 2505 |
4 |
# |
|
|
5 |
# Koha is free software; you can redistribute it and/or modify it |
6 |
# under the terms of the GNU General Public License as published by |
7 |
# the Free Software Foundation; either version 3 of the License, or |
8 |
# (at your option) any later version. |
9 |
# |
10 |
# Koha is distributed in the hope that it will be useful, but |
11 |
# WITHOUT ANY WARRANTY; without even the implied warranty of |
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 |
# GNU General Public License for more details. |
14 |
# |
15 |
# You should have received a copy of the GNU General Public License |
16 |
# along with Koha; if not, see <http://www.gnu.org/licenses>. |
17 |
|
18 |
use Modern::Perl; |
5 |
|
19 |
|
6 |
use C4::Context; |
20 |
use C4::Context; |
7 |
use Getopt::Long; |
21 |
use Getopt::Long; |
Lines 34-43
my $skip_index;
Link Here
|
34 |
my $reset; |
48 |
my $reset; |
35 |
my $biblios; |
49 |
my $biblios; |
36 |
my $authorities; |
50 |
my $authorities; |
37 |
my $noxml; |
51 |
my $as_usmarc; |
|
|
52 |
my $as_xml; |
38 |
my $noshadow; |
53 |
my $noshadow; |
39 |
my $want_help; |
54 |
my $want_help; |
40 |
my $as_xml; |
|
|
41 |
my $process_zebraqueue; |
55 |
my $process_zebraqueue; |
42 |
my $process_zebraqueue_skip_deletes; |
56 |
my $process_zebraqueue_skip_deletes; |
43 |
my $do_not_clear_zebraqueue; |
57 |
my $do_not_clear_zebraqueue; |
Lines 62-68
my $result = GetOptions(
Link Here
|
62 |
'I|skip-index' => \$skip_index, |
76 |
'I|skip-index' => \$skip_index, |
63 |
'nosanitize' => \$nosanitize, |
77 |
'nosanitize' => \$nosanitize, |
64 |
'b' => \$biblios, |
78 |
'b' => \$biblios, |
65 |
'noxml' => \$noxml, |
79 |
'noxml' => \$as_usmarc, |
66 |
'w' => \$noshadow, |
80 |
'w' => \$noshadow, |
67 |
'a' => \$authorities, |
81 |
'a' => \$authorities, |
68 |
'h|help' => \$want_help, |
82 |
'h|help' => \$want_help, |
Lines 84-89
if (not $result or $want_help) {
Link Here
|
84 |
exit 0; |
98 |
exit 0; |
85 |
} |
99 |
} |
86 |
|
100 |
|
|
|
101 |
if ( $as_xml ) { |
102 |
warn "Warning: You passed -x which is already the default and is now deprecated·\n"; |
103 |
} |
104 |
|
87 |
if( not defined $run_as_root and $run_user eq 'root') { |
105 |
if( not defined $run_as_root and $run_user eq 'root') { |
88 |
my $msg = "Warning: You are running this script as the user 'root'.\n"; |
106 |
my $msg = "Warning: You are running this script as the user 'root'.\n"; |
89 |
$msg .= "If this is intentional you must explicitly specify this using the -run-as-root switch\n"; |
107 |
$msg .= "If this is intentional you must explicitly specify this using the -run-as-root switch\n"; |
Lines 91-97
if( not defined $run_as_root and $run_user eq 'root') {
Link Here
|
91 |
die $msg; |
109 |
die $msg; |
92 |
} |
110 |
} |
93 |
|
111 |
|
94 |
if ( !$as_xml and $nosanitize ) { |
112 |
if ( $as_usmarc and $nosanitize ) { |
95 |
my $msg = "Cannot specify both -no_xml and -nosanitize\n"; |
113 |
my $msg = "Cannot specify both -no_xml and -nosanitize\n"; |
96 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
114 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
97 |
die $msg; |
115 |
die $msg; |
Lines 277-289
if ($keep_export) {
Link Here
|
277 |
|
295 |
|
278 |
sub do_one_pass { |
296 |
sub do_one_pass { |
279 |
if ($authorities) { |
297 |
if ($authorities) { |
280 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
298 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
281 |
} else { |
299 |
} else { |
282 |
print "skipping authorities\n" if ( $verbose_logging ); |
300 |
print "skipping authorities\n" if ( $verbose_logging ); |
283 |
} |
301 |
} |
284 |
|
302 |
|
285 |
if ($biblios) { |
303 |
if ($biblios) { |
286 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
304 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
287 |
} else { |
305 |
} else { |
288 |
print "skipping biblios\n" if ( $verbose_logging ); |
306 |
print "skipping biblios\n" if ( $verbose_logging ); |
289 |
} |
307 |
} |
Lines 330-336
sub check_zebra_dirs {
Link Here
|
330 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
348 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
331 |
|
349 |
|
332 |
sub index_records { |
350 |
sub index_records { |
333 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
351 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
334 |
|
352 |
|
335 |
my $num_records_exported = 0; |
353 |
my $num_records_exported = 0; |
336 |
my $records_deleted = {}; |
354 |
my $records_deleted = {}; |
Lines 357-374
sub index_records {
Link Here
|
357 |
unless ( $process_zebraqueue_skip_deletes ) { |
375 |
unless ( $process_zebraqueue_skip_deletes ) { |
358 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
376 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
359 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
377 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
360 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml); |
378 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_usmarc); |
361 |
mark_zebraqueue_batch_done($entries); |
379 |
mark_zebraqueue_batch_done($entries); |
362 |
} |
380 |
} |
363 |
|
381 |
|
364 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
382 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
365 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
383 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
366 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted); |
384 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_usmarc, $records_deleted); |
367 |
mark_zebraqueue_batch_done($entries); |
385 |
mark_zebraqueue_batch_done($entries); |
368 |
|
386 |
|
369 |
} else { |
387 |
} else { |
370 |
my $sth = select_all_records($record_type); |
388 |
my $sth = select_all_records($record_type); |
371 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize); |
389 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_usmarc, $nosanitize); |
372 |
unless ($do_not_clear_zebraqueue) { |
390 |
unless ($do_not_clear_zebraqueue) { |
373 |
mark_all_zebraqueue_done($record_type); |
391 |
mark_all_zebraqueue_done($record_type); |
374 |
} |
392 |
} |
Lines 390-396
sub index_records {
Link Here
|
390 |
print "REINDEXING zebra\n"; |
408 |
print "REINDEXING zebra\n"; |
391 |
print "====================\n"; |
409 |
print "====================\n"; |
392 |
} |
410 |
} |
393 |
my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ; |
411 |
my $record_fmt = ($as_usmarc) ? 'iso2709' : 'marcxml' ; |
394 |
if ($process_zebraqueue) { |
412 |
if ($process_zebraqueue) { |
395 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
413 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
396 |
if %$records_deleted; |
414 |
if %$records_deleted; |
Lines 470-494
sub select_all_biblios {
Link Here
|
470 |
return $sth; |
488 |
return $sth; |
471 |
} |
489 |
} |
472 |
|
490 |
|
473 |
sub include_xml_wrapper { |
|
|
474 |
my $as_xml = shift; |
475 |
my $record_type = shift; |
476 |
|
477 |
return 0 unless $as_xml; |
478 |
return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom'; |
479 |
return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom'; |
480 |
return 0; |
481 |
|
482 |
} |
483 |
|
484 |
sub export_marc_records_from_sth { |
491 |
sub export_marc_records_from_sth { |
485 |
my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_; |
492 |
my ($record_type, $sth, $directory, $as_usmarc, $nosanitize) = @_; |
486 |
|
493 |
|
487 |
my $num_exported = 0; |
494 |
my $num_exported = 0; |
488 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
495 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
489 |
|
496 |
|
490 |
print {$fh} $marcxml_open |
497 |
print {$fh} $marcxml_open |
491 |
if include_xml_wrapper($as_xml, $record_type); |
498 |
unless $as_usmarc; |
492 |
|
499 |
|
493 |
my $i = 0; |
500 |
my $i = 0; |
494 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
501 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
Lines 532-542
sub export_marc_records_from_sth {
Link Here
|
532 |
} |
539 |
} |
533 |
next; |
540 |
next; |
534 |
} |
541 |
} |
535 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml); |
542 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
536 |
if (defined $marc) { |
543 |
if (defined $marc) { |
537 |
eval { |
544 |
eval { |
538 |
my $rec; |
545 |
my $rec; |
539 |
if ($as_xml) { |
546 |
if ($as_usmarc) { |
|
|
547 |
$rec = $marc->as_usmarc(); |
548 |
} else { |
540 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
549 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
541 |
eval { |
550 |
eval { |
542 |
my $doc = $tester->parse_string($rec); |
551 |
my $doc = $tester->parse_string($rec); |
Lines 545-577
sub export_marc_records_from_sth {
Link Here
|
545 |
die "invalid XML: $@"; |
554 |
die "invalid XML: $@"; |
546 |
} |
555 |
} |
547 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
556 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
548 |
} else { |
|
|
549 |
$rec = $marc->as_usmarc(); |
550 |
} |
557 |
} |
551 |
print {$fh} $rec; |
558 |
print {$fh} $rec; |
552 |
$num_exported++; |
559 |
$num_exported++; |
553 |
}; |
560 |
}; |
554 |
if ($@) { |
561 |
if ($@) { |
555 |
warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML"); |
562 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
556 |
warn "... specific error is $@" if $verbose_logging; |
563 |
warn "... specific error is $@" if $verbose_logging; |
557 |
} |
564 |
} |
558 |
} |
565 |
} |
559 |
} |
566 |
} |
560 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
567 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
561 |
print {$fh} $marcxml_close |
568 |
print {$fh} $marcxml_close |
562 |
if include_xml_wrapper($as_xml, $record_type); |
569 |
unless $as_usmarc; |
|
|
570 |
|
563 |
close $fh; |
571 |
close $fh; |
564 |
return $num_exported; |
572 |
return $num_exported; |
565 |
} |
573 |
} |
566 |
|
574 |
|
567 |
sub export_marc_records_from_list { |
575 |
sub export_marc_records_from_list { |
568 |
my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_; |
576 |
my ($record_type, $entries, $directory, $as_usmarc, $records_deleted) = @_; |
569 |
|
577 |
|
570 |
my $num_exported = 0; |
578 |
my $num_exported = 0; |
571 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
579 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
572 |
|
580 |
|
573 |
print {$fh} $marcxml_open |
581 |
print {$fh} $marcxml_open |
574 |
if include_xml_wrapper($as_xml, $record_type); |
582 |
unless $as_usmarc; |
575 |
|
583 |
|
576 |
my $i = 0; |
584 |
my $i = 0; |
577 |
|
585 |
|
Lines 582-622
sub export_marc_records_from_list {
Link Here
|
582 |
@$entries ) { |
590 |
@$entries ) { |
583 |
print "." if ( $verbose_logging ); |
591 |
print "." if ( $verbose_logging ); |
584 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
592 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
585 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml); |
593 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
586 |
if (defined $marc) { |
594 |
if (defined $marc) { |
587 |
eval { |
595 |
eval { |
588 |
my $rec; |
596 |
my $rec; |
589 |
if ($as_xml) { |
597 |
if ( $as_usmarc ) { |
|
|
598 |
$rec = $marc->as_usmarc(); |
599 |
} else { |
590 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
600 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
591 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
601 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
592 |
} else { |
|
|
593 |
$rec = $marc->as_usmarc(); |
594 |
} |
602 |
} |
595 |
print {$fh} $rec; |
603 |
print {$fh} $rec; |
596 |
$num_exported++; |
604 |
$num_exported++; |
597 |
}; |
605 |
}; |
598 |
if ($@) { |
606 |
if ($@) { |
599 |
warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML"); |
607 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
600 |
} |
608 |
} |
601 |
} |
609 |
} |
602 |
} |
610 |
} |
603 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
611 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
604 |
|
612 |
|
605 |
print {$fh} $marcxml_close |
613 |
print {$fh} $marcxml_close |
606 |
if include_xml_wrapper($as_xml, $record_type); |
614 |
unless $as_usmarc; |
607 |
|
615 |
|
608 |
close $fh; |
616 |
close $fh; |
609 |
return $num_exported; |
617 |
return $num_exported; |
610 |
} |
618 |
} |
611 |
|
619 |
|
612 |
sub generate_deleted_marc_records { |
620 |
sub generate_deleted_marc_records { |
613 |
my ($record_type, $entries, $directory, $as_xml) = @_; |
621 |
|
|
|
622 |
my ($record_type, $entries, $directory, $as_usmarc) = @_; |
614 |
|
623 |
|
615 |
my $records_deleted = {}; |
624 |
my $records_deleted = {}; |
616 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
625 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
617 |
|
626 |
|
618 |
print {$fh} $marcxml_open |
627 |
print {$fh} $marcxml_open |
619 |
if include_xml_wrapper($as_xml, $record_type); |
628 |
unless $as_usmarc; |
620 |
|
629 |
|
621 |
my $i = 0; |
630 |
my $i = 0; |
622 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
631 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
Lines 634-644
sub generate_deleted_marc_records {
Link Here
|
634 |
} |
643 |
} |
635 |
|
644 |
|
636 |
my $rec; |
645 |
my $rec; |
637 |
if ($as_xml) { |
646 |
if ( $as_usmarc ) { |
|
|
647 |
$rec = $marc->as_usmarc(); |
648 |
} else { |
638 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
649 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
|
|
650 |
# Remove the record's XML header |
639 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
651 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
640 |
} else { |
|
|
641 |
$rec = $marc->as_usmarc(); |
642 |
} |
652 |
} |
643 |
print {$fh} $rec; |
653 |
print {$fh} $rec; |
644 |
|
654 |
|
Lines 647-664
sub generate_deleted_marc_records {
Link Here
|
647 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
657 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
648 |
|
658 |
|
649 |
print {$fh} $marcxml_close |
659 |
print {$fh} $marcxml_close |
650 |
if include_xml_wrapper($as_xml, $record_type); |
660 |
unless $as_usmarc; |
651 |
|
661 |
|
652 |
close $fh; |
662 |
close $fh; |
653 |
return $records_deleted; |
663 |
return $records_deleted; |
654 |
|
|
|
655 |
|
656 |
} |
664 |
} |
657 |
|
665 |
|
658 |
sub get_corrected_marc_record { |
666 |
sub get_corrected_marc_record { |
659 |
my ($record_type, $record_number, $noxml) = @_; |
667 |
my ($record_type, $record_number, $as_usmarc) = @_; |
660 |
|
668 |
|
661 |
my $marc = get_raw_marc_record($record_type, $record_number, $noxml); |
669 |
my $marc = get_raw_marc_record($record_type, $record_number, $as_usmarc); |
662 |
|
670 |
|
663 |
if (defined $marc) { |
671 |
if (defined $marc) { |
664 |
fix_leader($marc); |
672 |
fix_leader($marc); |
Lines 677-687
sub get_corrected_marc_record {
Link Here
|
677 |
} |
685 |
} |
678 |
|
686 |
|
679 |
sub get_raw_marc_record { |
687 |
sub get_raw_marc_record { |
680 |
my ($record_type, $record_number, $noxml) = @_; |
688 |
my ($record_type, $record_number, $as_usmarc) = @_; |
681 |
|
689 |
|
682 |
my $marc; |
690 |
my $marc; |
683 |
if ($record_type eq 'biblio') { |
691 |
if ($record_type eq 'biblio') { |
684 |
if ($noxml) { |
692 |
if ($as_usmarc) { |
685 |
my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?"); |
693 |
my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?"); |
686 |
$fetch_sth->execute($record_number); |
694 |
$fetch_sth->execute($record_number); |
687 |
if (my ($blob) = $fetch_sth->fetchrow_array) { |
695 |
if (my ($blob) = $fetch_sth->fetchrow_array) { |
Lines 908-916
Parameters:
Link Here
|
908 |
option is recommended only |
916 |
option is recommended only |
909 |
for advanced user. |
917 |
for advanced user. |
910 |
|
918 |
|
911 |
-x export and index as xml instead of is02709 (biblios only). |
|
|
912 |
use this if you might have records > 99,999 chars, |
913 |
|
914 |
-nosanitize export biblio/authority records directly from DB marcxml |
919 |
-nosanitize export biblio/authority records directly from DB marcxml |
915 |
field without sanitizing records. It speed up |
920 |
field without sanitizing records. It speed up |
916 |
dump process but could fail if DB contains badly |
921 |
dump process but could fail if DB contains badly |
917 |
- |
|
|