Lines 48-54
my $skip_index;
Link Here
|
48 |
my $reset; |
48 |
my $reset; |
49 |
my $biblios; |
49 |
my $biblios; |
50 |
my $authorities; |
50 |
my $authorities; |
51 |
my $as_usmarc; |
|
|
52 |
my $as_xml; |
51 |
my $as_xml; |
53 |
my $noshadow; |
52 |
my $noshadow; |
54 |
my $want_help; |
53 |
my $want_help; |
Lines 76-82
my $result = GetOptions(
Link Here
|
76 |
'I|skip-index' => \$skip_index, |
75 |
'I|skip-index' => \$skip_index, |
77 |
'nosanitize' => \$nosanitize, |
76 |
'nosanitize' => \$nosanitize, |
78 |
'b' => \$biblios, |
77 |
'b' => \$biblios, |
79 |
'noxml' => \$as_usmarc, |
|
|
80 |
'w' => \$noshadow, |
78 |
'w' => \$noshadow, |
81 |
'a' => \$authorities, |
79 |
'a' => \$authorities, |
82 |
'h|help' => \$want_help, |
80 |
'h|help' => \$want_help, |
Lines 110-121
if( not defined $run_as_root and $run_user eq 'root') {
Link Here
|
110 |
die $msg; |
108 |
die $msg; |
111 |
} |
109 |
} |
112 |
|
110 |
|
113 |
if ( $as_usmarc and $nosanitize ) { |
|
|
114 |
my $msg = "Cannot specify both -noxml and -nosanitize\n"; |
115 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
116 |
die $msg; |
117 |
} |
118 |
|
119 |
if ($process_zebraqueue and ($skip_export or $reset)) { |
111 |
if ($process_zebraqueue and ($skip_export or $reset)) { |
120 |
my $msg = "Cannot specify -r or -s if -z is specified\n"; |
112 |
my $msg = "Cannot specify -r or -s if -z is specified\n"; |
121 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
113 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
Lines 295-307
if ($keep_export) {
Link Here
|
295 |
|
287 |
|
296 |
sub do_one_pass { |
288 |
sub do_one_pass { |
297 |
if ($authorities) { |
289 |
if ($authorities) { |
298 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
290 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
299 |
} else { |
291 |
} else { |
300 |
print "skipping authorities\n" if ( $verbose_logging ); |
292 |
print "skipping authorities\n" if ( $verbose_logging ); |
301 |
} |
293 |
} |
302 |
|
294 |
|
303 |
if ($biblios) { |
295 |
if ($biblios) { |
304 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
296 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
305 |
} else { |
297 |
} else { |
306 |
print "skipping biblios\n" if ( $verbose_logging ); |
298 |
print "skipping biblios\n" if ( $verbose_logging ); |
307 |
} |
299 |
} |
Lines 348-354
sub check_zebra_dirs {
Link Here
|
348 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
340 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
349 |
|
341 |
|
350 |
sub index_records { |
342 |
sub index_records { |
351 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
343 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
352 |
|
344 |
|
353 |
my $num_records_exported = 0; |
345 |
my $num_records_exported = 0; |
354 |
my $records_deleted = {}; |
346 |
my $records_deleted = {}; |
Lines 375-392
sub index_records {
Link Here
|
375 |
unless ( $process_zebraqueue_skip_deletes ) { |
367 |
unless ( $process_zebraqueue_skip_deletes ) { |
376 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
368 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
377 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
369 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
378 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_usmarc); |
370 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type"); |
379 |
mark_zebraqueue_batch_done($entries); |
371 |
mark_zebraqueue_batch_done($entries); |
380 |
} |
372 |
} |
381 |
|
373 |
|
382 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
374 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
383 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
375 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
384 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_usmarc, $records_deleted); |
376 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $records_deleted); |
385 |
mark_zebraqueue_batch_done($entries); |
377 |
mark_zebraqueue_batch_done($entries); |
386 |
|
378 |
|
387 |
} else { |
379 |
} else { |
388 |
my $sth = select_all_records($record_type); |
380 |
my $sth = select_all_records($record_type); |
389 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_usmarc, $nosanitize); |
381 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize); |
390 |
unless ($do_not_clear_zebraqueue) { |
382 |
unless ($do_not_clear_zebraqueue) { |
391 |
mark_all_zebraqueue_done($record_type); |
383 |
mark_all_zebraqueue_done($record_type); |
392 |
} |
384 |
} |
Lines 408-414
sub index_records {
Link Here
|
408 |
print "REINDEXING zebra\n"; |
400 |
print "REINDEXING zebra\n"; |
409 |
print "====================\n"; |
401 |
print "====================\n"; |
410 |
} |
402 |
} |
411 |
my $record_fmt = ($as_usmarc) ? 'iso2709' : 'marcxml' ; |
403 |
my $record_fmt = 'marcxml'; |
412 |
if ($process_zebraqueue) { |
404 |
if ($process_zebraqueue) { |
413 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
405 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
414 |
if %$records_deleted; |
406 |
if %$records_deleted; |
Lines 489-501
sub select_all_biblios {
Link Here
|
489 |
} |
481 |
} |
490 |
|
482 |
|
491 |
sub export_marc_records_from_sth { |
483 |
sub export_marc_records_from_sth { |
492 |
my ($record_type, $sth, $directory, $as_usmarc, $nosanitize) = @_; |
484 |
my ($record_type, $sth, $directory, $nosanitize) = @_; |
493 |
|
485 |
|
494 |
my $num_exported = 0; |
486 |
my $num_exported = 0; |
495 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
487 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
496 |
|
488 |
|
497 |
print {$fh} $marcxml_open |
489 |
print {$fh} $marcxml_open; |
498 |
unless $as_usmarc; |
|
|
499 |
|
490 |
|
500 |
my $i = 0; |
491 |
my $i = 0; |
501 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
492 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
Lines 539-585
sub export_marc_records_from_sth {
Link Here
|
539 |
} |
530 |
} |
540 |
next; |
531 |
next; |
541 |
} |
532 |
} |
542 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
533 |
my ($marc) = get_corrected_marc_record($record_type, $record_number); |
543 |
if (defined $marc) { |
534 |
if (defined $marc) { |
544 |
eval { |
535 |
eval { |
545 |
my $rec; |
536 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
546 |
if ($as_usmarc) { |
537 |
eval { |
547 |
$rec = $marc->as_usmarc(); |
538 |
my $doc = $tester->parse_string($rec); |
548 |
} else { |
539 |
}; |
549 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
540 |
if ($@) { |
550 |
eval { |
541 |
die "invalid XML: $@"; |
551 |
my $doc = $tester->parse_string($rec); |
|
|
552 |
}; |
553 |
if ($@) { |
554 |
die "invalid XML: $@"; |
555 |
} |
556 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
557 |
} |
542 |
} |
|
|
543 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
558 |
print {$fh} $rec; |
544 |
print {$fh} $rec; |
559 |
$num_exported++; |
545 |
$num_exported++; |
560 |
}; |
546 |
}; |
561 |
if ($@) { |
547 |
if ($@) { |
562 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
548 |
warn "Error exporting record $record_number ($record_type) XML"; |
563 |
warn "... specific error is $@" if $verbose_logging; |
549 |
warn "... specific error is $@" if $verbose_logging; |
564 |
} |
550 |
} |
565 |
} |
551 |
} |
566 |
} |
552 |
} |
567 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
553 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
568 |
print {$fh} $marcxml_close |
554 |
print {$fh} $marcxml_close; |
569 |
unless $as_usmarc; |
|
|
570 |
|
555 |
|
571 |
close $fh; |
556 |
close $fh; |
572 |
return $num_exported; |
557 |
return $num_exported; |
573 |
} |
558 |
} |
574 |
|
559 |
|
575 |
sub export_marc_records_from_list { |
560 |
sub export_marc_records_from_list { |
576 |
my ($record_type, $entries, $directory, $as_usmarc, $records_deleted) = @_; |
561 |
my ($record_type, $entries, $directory, $records_deleted) = @_; |
577 |
|
562 |
|
578 |
my $num_exported = 0; |
563 |
my $num_exported = 0; |
579 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
564 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
580 |
|
565 |
|
581 |
print {$fh} $marcxml_open |
566 |
print {$fh} $marcxml_open; |
582 |
unless $as_usmarc; |
|
|
583 |
|
567 |
|
584 |
my $i = 0; |
568 |
my $i = 0; |
585 |
|
569 |
|
Lines 590-617
sub export_marc_records_from_list {
Link Here
|
590 |
@$entries ) { |
574 |
@$entries ) { |
591 |
print "." if ( $verbose_logging ); |
575 |
print "." if ( $verbose_logging ); |
592 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
576 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
593 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
577 |
my ($marc) = get_corrected_marc_record($record_type, $record_number); |
594 |
if (defined $marc) { |
578 |
if (defined $marc) { |
595 |
eval { |
579 |
eval { |
596 |
my $rec; |
580 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
597 |
if ( $as_usmarc ) { |
581 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
598 |
$rec = $marc->as_usmarc(); |
|
|
599 |
} else { |
600 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
601 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
602 |
} |
603 |
print {$fh} $rec; |
582 |
print {$fh} $rec; |
604 |
$num_exported++; |
583 |
$num_exported++; |
605 |
}; |
584 |
}; |
606 |
if ($@) { |
585 |
if ($@) { |
607 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
586 |
warn "Error exporting record $record_number ($record_type) XML"; |
608 |
} |
587 |
} |
609 |
} |
588 |
} |
610 |
} |
589 |
} |
611 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
590 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
612 |
|
591 |
|
613 |
print {$fh} $marcxml_close |
592 |
print {$fh} $marcxml_close; |
614 |
unless $as_usmarc; |
|
|
615 |
|
593 |
|
616 |
close $fh; |
594 |
close $fh; |
617 |
return $num_exported; |
595 |
return $num_exported; |
Lines 619-631
sub export_marc_records_from_list {
Link Here
|
619 |
|
597 |
|
620 |
sub generate_deleted_marc_records { |
598 |
sub generate_deleted_marc_records { |
621 |
|
599 |
|
622 |
my ($record_type, $entries, $directory, $as_usmarc) = @_; |
600 |
my ($record_type, $entries, $directory) = @_; |
623 |
|
601 |
|
624 |
my $records_deleted = {}; |
602 |
my $records_deleted = {}; |
625 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
603 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
626 |
|
604 |
|
627 |
print {$fh} $marcxml_open |
605 |
print {$fh} $marcxml_open; |
628 |
unless $as_usmarc; |
|
|
629 |
|
606 |
|
630 |
my $i = 0; |
607 |
my $i = 0; |
631 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
608 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
Lines 642-672
sub generate_deleted_marc_records {
Link Here
|
642 |
fix_unimarc_100($marc); |
619 |
fix_unimarc_100($marc); |
643 |
} |
620 |
} |
644 |
|
621 |
|
645 |
my $rec; |
622 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
646 |
if ( $as_usmarc ) { |
623 |
# Remove the record's XML header |
647 |
$rec = $marc->as_usmarc(); |
624 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
648 |
} else { |
|
|
649 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
650 |
# Remove the record's XML header |
651 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
652 |
} |
653 |
print {$fh} $rec; |
625 |
print {$fh} $rec; |
654 |
|
626 |
|
655 |
$records_deleted->{$record_number} = 1; |
627 |
$records_deleted->{$record_number} = 1; |
656 |
} |
628 |
} |
657 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
629 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
658 |
|
630 |
|
659 |
print {$fh} $marcxml_close |
631 |
print {$fh} $marcxml_close; |
660 |
unless $as_usmarc; |
|
|
661 |
|
632 |
|
662 |
close $fh; |
633 |
close $fh; |
663 |
return $records_deleted; |
634 |
return $records_deleted; |
664 |
} |
635 |
} |
665 |
|
636 |
|
666 |
sub get_corrected_marc_record { |
637 |
sub get_corrected_marc_record { |
667 |
my ($record_type, $record_number, $as_usmarc) = @_; |
638 |
my ($record_type, $record_number) = @_; |
668 |
|
639 |
|
669 |
my $marc = get_raw_marc_record($record_type, $record_number, $as_usmarc); |
640 |
my $marc = get_raw_marc_record($record_type, $record_number); |
670 |
|
641 |
|
671 |
if (defined $marc) { |
642 |
if (defined $marc) { |
672 |
fix_leader($marc); |
643 |
fix_leader($marc); |
Lines 685-718
sub get_corrected_marc_record {
Link Here
|
685 |
} |
656 |
} |
686 |
|
657 |
|
687 |
sub get_raw_marc_record { |
658 |
sub get_raw_marc_record { |
688 |
my ($record_type, $record_number, $as_usmarc) = @_; |
659 |
my ($record_type, $record_number) = @_; |
689 |
|
660 |
|
690 |
my $marc; |
661 |
my $marc; |
691 |
if ($record_type eq 'biblio') { |
662 |
if ($record_type eq 'biblio') { |
692 |
if ($as_usmarc) { |
663 |
eval { $marc = GetMarcBiblio($record_number, 1); }; |
693 |
my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?"); |
664 |
if ($@ || !$marc) { |
694 |
$fetch_sth->execute($record_number); |
665 |
# here we do warn since catching an exception |
695 |
if (my ($blob) = $fetch_sth->fetchrow_array) { |
666 |
# means that the bib was found but failed |
696 |
$marc = MARC::Record->new_from_usmarc($blob); |
667 |
# to be parsed |
697 |
unless ($marc) { |
668 |
warn "error retrieving biblio $record_number"; |
698 |
warn "error creating MARC::Record from $blob"; |
669 |
return; |
699 |
} |
|
|
700 |
} |
701 |
# failure to find a bib is not a problem - |
702 |
# a delete could have been done before |
703 |
# trying to process a record update |
704 |
|
705 |
$fetch_sth->finish(); |
706 |
return unless $marc; |
707 |
} else { |
708 |
eval { $marc = GetMarcBiblio($record_number, 1); }; |
709 |
if ($@ || !$marc) { |
710 |
# here we do warn since catching an exception |
711 |
# means that the bib was found but failed |
712 |
# to be parsed |
713 |
warn "error retrieving biblio $record_number"; |
714 |
return; |
715 |
} |
716 |
} |
670 |
} |
717 |
} else { |
671 |
} else { |
718 |
eval { $marc = GetAuthority($record_number); }; |
672 |
eval { $marc = GetAuthority($record_number); }; |
Lines 918-928
Parameters:
Link Here
|
918 |
already exported the records |
872 |
already exported the records |
919 |
in a previous run. |
873 |
in a previous run. |
920 |
|
874 |
|
921 |
-noxml index from ISO MARC blob |
|
|
922 |
instead of MARC XML. This |
923 |
option is recommended only |
924 |
for advanced user. |
925 |
|
926 |
-nosanitize export biblio/authority records directly from DB marcxml |
875 |
-nosanitize export biblio/authority records directly from DB marcxml |
927 |
field without sanitizing records. It speed up |
876 |
field without sanitizing records. It speed up |
928 |
dump process but could fail if DB contains badly |
877 |
dump process but could fail if DB contains badly |
929 |
- |
|
|