|
Lines 48-54
my $skip_index;
Link Here
|
| 48 |
my $reset; |
48 |
my $reset; |
| 49 |
my $biblios; |
49 |
my $biblios; |
| 50 |
my $authorities; |
50 |
my $authorities; |
| 51 |
my $as_usmarc; |
|
|
| 52 |
my $as_xml; |
51 |
my $as_xml; |
| 53 |
my $noshadow; |
52 |
my $noshadow; |
| 54 |
my $want_help; |
53 |
my $want_help; |
|
Lines 76-82
my $result = GetOptions(
Link Here
|
| 76 |
'I|skip-index' => \$skip_index, |
75 |
'I|skip-index' => \$skip_index, |
| 77 |
'nosanitize' => \$nosanitize, |
76 |
'nosanitize' => \$nosanitize, |
| 78 |
'b' => \$biblios, |
77 |
'b' => \$biblios, |
| 79 |
'noxml' => \$as_usmarc, |
|
|
| 80 |
'w' => \$noshadow, |
78 |
'w' => \$noshadow, |
| 81 |
'a' => \$authorities, |
79 |
'a' => \$authorities, |
| 82 |
'h|help' => \$want_help, |
80 |
'h|help' => \$want_help, |
|
Lines 110-121
if( not defined $run_as_root and $run_user eq 'root') {
Link Here
|
| 110 |
die $msg; |
108 |
die $msg; |
| 111 |
} |
109 |
} |
| 112 |
|
110 |
|
| 113 |
if ( $as_usmarc and $nosanitize ) { |
|
|
| 114 |
my $msg = "Cannot specify both -noxml and -nosanitize\n"; |
| 115 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
| 116 |
die $msg; |
| 117 |
} |
| 118 |
|
| 119 |
if ($process_zebraqueue and ($skip_export or $reset)) { |
111 |
if ($process_zebraqueue and ($skip_export or $reset)) { |
| 120 |
my $msg = "Cannot specify -r or -s if -z is specified\n"; |
112 |
my $msg = "Cannot specify -r or -s if -z is specified\n"; |
| 121 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
113 |
$msg .= "Please do '$0 --help' to see usage.\n"; |
|
Lines 295-307
if ($keep_export) {
Link Here
|
| 295 |
|
287 |
|
| 296 |
sub do_one_pass { |
288 |
sub do_one_pass { |
| 297 |
if ($authorities) { |
289 |
if ($authorities) { |
| 298 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
290 |
index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); |
| 299 |
} else { |
291 |
} else { |
| 300 |
print "skipping authorities\n" if ( $verbose_logging ); |
292 |
print "skipping authorities\n" if ( $verbose_logging ); |
| 301 |
} |
293 |
} |
| 302 |
|
294 |
|
| 303 |
if ($biblios) { |
295 |
if ($biblios) { |
| 304 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
296 |
index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); |
| 305 |
} else { |
297 |
} else { |
| 306 |
print "skipping biblios\n" if ( $verbose_logging ); |
298 |
print "skipping biblios\n" if ( $verbose_logging ); |
| 307 |
} |
299 |
} |
|
Lines 348-354
sub check_zebra_dirs {
Link Here
|
| 348 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
340 |
} # ---------- end of subroutine check_zebra_dirs ---------- |
| 349 |
|
341 |
|
| 350 |
sub index_records { |
342 |
sub index_records { |
| 351 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_usmarc, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
343 |
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; |
| 352 |
|
344 |
|
| 353 |
my $num_records_exported = 0; |
345 |
my $num_records_exported = 0; |
| 354 |
my $records_deleted = {}; |
346 |
my $records_deleted = {}; |
|
Lines 375-392
sub index_records {
Link Here
|
| 375 |
unless ( $process_zebraqueue_skip_deletes ) { |
367 |
unless ( $process_zebraqueue_skip_deletes ) { |
| 376 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
368 |
$entries = select_zebraqueue_records($record_type, 'deleted'); |
| 377 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
369 |
mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); |
| 378 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_usmarc); |
370 |
$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type"); |
| 379 |
mark_zebraqueue_batch_done($entries); |
371 |
mark_zebraqueue_batch_done($entries); |
| 380 |
} |
372 |
} |
| 381 |
|
373 |
|
| 382 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
374 |
$entries = select_zebraqueue_records($record_type, 'updated'); |
| 383 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
375 |
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); |
| 384 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_usmarc, $records_deleted); |
376 |
$num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $records_deleted); |
| 385 |
mark_zebraqueue_batch_done($entries); |
377 |
mark_zebraqueue_batch_done($entries); |
| 386 |
|
378 |
|
| 387 |
} else { |
379 |
} else { |
| 388 |
my $sth = select_all_records($record_type); |
380 |
my $sth = select_all_records($record_type); |
| 389 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_usmarc, $nosanitize); |
381 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize); |
| 390 |
unless ($do_not_clear_zebraqueue) { |
382 |
unless ($do_not_clear_zebraqueue) { |
| 391 |
mark_all_zebraqueue_done($record_type); |
383 |
mark_all_zebraqueue_done($record_type); |
| 392 |
} |
384 |
} |
|
Lines 408-414
sub index_records {
Link Here
|
| 408 |
print "REINDEXING zebra\n"; |
400 |
print "REINDEXING zebra\n"; |
| 409 |
print "====================\n"; |
401 |
print "====================\n"; |
| 410 |
} |
402 |
} |
| 411 |
my $record_fmt = ($as_usmarc) ? 'iso2709' : 'marcxml' ; |
403 |
my $record_fmt = 'marcxml'; |
| 412 |
if ($process_zebraqueue) { |
404 |
if ($process_zebraqueue) { |
| 413 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
405 |
do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) |
| 414 |
if %$records_deleted; |
406 |
if %$records_deleted; |
|
Lines 489-501
sub select_all_biblios {
Link Here
|
| 489 |
} |
481 |
} |
| 490 |
|
482 |
|
| 491 |
sub export_marc_records_from_sth { |
483 |
sub export_marc_records_from_sth { |
| 492 |
my ($record_type, $sth, $directory, $as_usmarc, $nosanitize) = @_; |
484 |
my ($record_type, $sth, $directory, $nosanitize) = @_; |
| 493 |
|
485 |
|
| 494 |
my $num_exported = 0; |
486 |
my $num_exported = 0; |
| 495 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
487 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
| 496 |
|
488 |
|
| 497 |
print {$fh} $marcxml_open |
489 |
print {$fh} $marcxml_open; |
| 498 |
unless $as_usmarc; |
|
|
| 499 |
|
490 |
|
| 500 |
my $i = 0; |
491 |
my $i = 0; |
| 501 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
492 |
my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); |
|
Lines 539-585
sub export_marc_records_from_sth {
Link Here
|
| 539 |
} |
530 |
} |
| 540 |
next; |
531 |
next; |
| 541 |
} |
532 |
} |
| 542 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
533 |
my ($marc) = get_corrected_marc_record($record_type, $record_number); |
| 543 |
if (defined $marc) { |
534 |
if (defined $marc) { |
| 544 |
eval { |
535 |
eval { |
| 545 |
my $rec; |
536 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
| 546 |
if ($as_usmarc) { |
537 |
eval { |
| 547 |
$rec = $marc->as_usmarc(); |
538 |
my $doc = $tester->parse_string($rec); |
| 548 |
} else { |
539 |
}; |
| 549 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
540 |
if ($@) { |
| 550 |
eval { |
541 |
die "invalid XML: $@"; |
| 551 |
my $doc = $tester->parse_string($rec); |
|
|
| 552 |
}; |
| 553 |
if ($@) { |
| 554 |
die "invalid XML: $@"; |
| 555 |
} |
| 556 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 557 |
} |
542 |
} |
|
|
543 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 558 |
print {$fh} $rec; |
544 |
print {$fh} $rec; |
| 559 |
$num_exported++; |
545 |
$num_exported++; |
| 560 |
}; |
546 |
}; |
| 561 |
if ($@) { |
547 |
if ($@) { |
| 562 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
548 |
warn "Error exporting record $record_number ($record_type) XML"; |
| 563 |
warn "... specific error is $@" if $verbose_logging; |
549 |
warn "... specific error is $@" if $verbose_logging; |
| 564 |
} |
550 |
} |
| 565 |
} |
551 |
} |
| 566 |
} |
552 |
} |
| 567 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
553 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
| 568 |
print {$fh} $marcxml_close |
554 |
print {$fh} $marcxml_close; |
| 569 |
unless $as_usmarc; |
|
|
| 570 |
|
555 |
|
| 571 |
close $fh; |
556 |
close $fh; |
| 572 |
return $num_exported; |
557 |
return $num_exported; |
| 573 |
} |
558 |
} |
| 574 |
|
559 |
|
| 575 |
sub export_marc_records_from_list { |
560 |
sub export_marc_records_from_list { |
| 576 |
my ($record_type, $entries, $directory, $as_usmarc, $records_deleted) = @_; |
561 |
my ($record_type, $entries, $directory, $records_deleted) = @_; |
| 577 |
|
562 |
|
| 578 |
my $num_exported = 0; |
563 |
my $num_exported = 0; |
| 579 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
564 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
| 580 |
|
565 |
|
| 581 |
print {$fh} $marcxml_open |
566 |
print {$fh} $marcxml_open; |
| 582 |
unless $as_usmarc; |
|
|
| 583 |
|
567 |
|
| 584 |
my $i = 0; |
568 |
my $i = 0; |
| 585 |
|
569 |
|
|
Lines 590-617
sub export_marc_records_from_list {
Link Here
|
| 590 |
@$entries ) { |
574 |
@$entries ) { |
| 591 |
print "." if ( $verbose_logging ); |
575 |
print "." if ( $verbose_logging ); |
| 592 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
576 |
print "\r$i" unless ($i++ %100 or !$verbose_logging); |
| 593 |
my ($marc) = get_corrected_marc_record($record_type, $record_number, $as_usmarc); |
577 |
my ($marc) = get_corrected_marc_record($record_type, $record_number); |
| 594 |
if (defined $marc) { |
578 |
if (defined $marc) { |
| 595 |
eval { |
579 |
eval { |
| 596 |
my $rec; |
580 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
| 597 |
if ( $as_usmarc ) { |
581 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 598 |
$rec = $marc->as_usmarc(); |
|
|
| 599 |
} else { |
| 600 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
| 601 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 602 |
} |
| 603 |
print {$fh} $rec; |
582 |
print {$fh} $rec; |
| 604 |
$num_exported++; |
583 |
$num_exported++; |
| 605 |
}; |
584 |
}; |
| 606 |
if ($@) { |
585 |
if ($@) { |
| 607 |
warn "Error exporting record $record_number ($record_type) ".($as_usmarc ? "not XML" : "XML"); |
586 |
warn "Error exporting record $record_number ($record_type) XML"; |
| 608 |
} |
587 |
} |
| 609 |
} |
588 |
} |
| 610 |
} |
589 |
} |
| 611 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
590 |
print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); |
| 612 |
|
591 |
|
| 613 |
print {$fh} $marcxml_close |
592 |
print {$fh} $marcxml_close; |
| 614 |
unless $as_usmarc; |
|
|
| 615 |
|
593 |
|
| 616 |
close $fh; |
594 |
close $fh; |
| 617 |
return $num_exported; |
595 |
return $num_exported; |
|
Lines 619-631
sub export_marc_records_from_list {
Link Here
|
| 619 |
|
597 |
|
| 620 |
sub generate_deleted_marc_records { |
598 |
sub generate_deleted_marc_records { |
| 621 |
|
599 |
|
| 622 |
my ($record_type, $entries, $directory, $as_usmarc) = @_; |
600 |
my ($record_type, $entries, $directory) = @_; |
| 623 |
|
601 |
|
| 624 |
my $records_deleted = {}; |
602 |
my $records_deleted = {}; |
| 625 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
603 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!; |
| 626 |
|
604 |
|
| 627 |
print {$fh} $marcxml_open |
605 |
print {$fh} $marcxml_open; |
| 628 |
unless $as_usmarc; |
|
|
| 629 |
|
606 |
|
| 630 |
my $i = 0; |
607 |
my $i = 0; |
| 631 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
608 |
foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { |
|
Lines 642-672
sub generate_deleted_marc_records {
Link Here
|
| 642 |
fix_unimarc_100($marc); |
619 |
fix_unimarc_100($marc); |
| 643 |
} |
620 |
} |
| 644 |
|
621 |
|
| 645 |
my $rec; |
622 |
my $rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
| 646 |
if ( $as_usmarc ) { |
623 |
# Remove the record's XML header |
| 647 |
$rec = $marc->as_usmarc(); |
624 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 648 |
} else { |
|
|
| 649 |
$rec = $marc->as_xml_record(C4::Context->preference('marcflavour')); |
| 650 |
# Remove the record's XML header |
| 651 |
$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!; |
| 652 |
} |
| 653 |
print {$fh} $rec; |
625 |
print {$fh} $rec; |
| 654 |
|
626 |
|
| 655 |
$records_deleted->{$record_number} = 1; |
627 |
$records_deleted->{$record_number} = 1; |
| 656 |
} |
628 |
} |
| 657 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
629 |
print "\nRecords exported: $i\n" if ( $verbose_logging ); |
| 658 |
|
630 |
|
| 659 |
print {$fh} $marcxml_close |
631 |
print {$fh} $marcxml_close; |
| 660 |
unless $as_usmarc; |
|
|
| 661 |
|
632 |
|
| 662 |
close $fh; |
633 |
close $fh; |
| 663 |
return $records_deleted; |
634 |
return $records_deleted; |
| 664 |
} |
635 |
} |
| 665 |
|
636 |
|
| 666 |
sub get_corrected_marc_record { |
637 |
sub get_corrected_marc_record { |
| 667 |
my ($record_type, $record_number, $as_usmarc) = @_; |
638 |
my ($record_type, $record_number) = @_; |
| 668 |
|
639 |
|
| 669 |
my $marc = get_raw_marc_record($record_type, $record_number, $as_usmarc); |
640 |
my $marc = get_raw_marc_record($record_type, $record_number); |
| 670 |
|
641 |
|
| 671 |
if (defined $marc) { |
642 |
if (defined $marc) { |
| 672 |
fix_leader($marc); |
643 |
fix_leader($marc); |
|
Lines 685-718
sub get_corrected_marc_record {
Link Here
|
| 685 |
} |
656 |
} |
| 686 |
|
657 |
|
| 687 |
sub get_raw_marc_record { |
658 |
sub get_raw_marc_record { |
| 688 |
my ($record_type, $record_number, $as_usmarc) = @_; |
659 |
my ($record_type, $record_number) = @_; |
| 689 |
|
660 |
|
| 690 |
my $marc; |
661 |
my $marc; |
| 691 |
if ($record_type eq 'biblio') { |
662 |
if ($record_type eq 'biblio') { |
| 692 |
if ($as_usmarc) { |
663 |
eval { $marc = GetMarcBiblio($record_number, 1); }; |
| 693 |
my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?"); |
664 |
if ($@ || !$marc) { |
| 694 |
$fetch_sth->execute($record_number); |
665 |
# here we do warn since catching an exception |
| 695 |
if (my ($blob) = $fetch_sth->fetchrow_array) { |
666 |
# means that the bib was found but failed |
| 696 |
$marc = MARC::Record->new_from_usmarc($blob); |
667 |
# to be parsed |
| 697 |
unless ($marc) { |
668 |
warn "error retrieving biblio $record_number"; |
| 698 |
warn "error creating MARC::Record from $blob"; |
669 |
return; |
| 699 |
} |
|
|
| 700 |
} |
| 701 |
# failure to find a bib is not a problem - |
| 702 |
# a delete could have been done before |
| 703 |
# trying to process a record update |
| 704 |
|
| 705 |
$fetch_sth->finish(); |
| 706 |
return unless $marc; |
| 707 |
} else { |
| 708 |
eval { $marc = GetMarcBiblio($record_number, 1); }; |
| 709 |
if ($@ || !$marc) { |
| 710 |
# here we do warn since catching an exception |
| 711 |
# means that the bib was found but failed |
| 712 |
# to be parsed |
| 713 |
warn "error retrieving biblio $record_number"; |
| 714 |
return; |
| 715 |
} |
| 716 |
} |
670 |
} |
| 717 |
} else { |
671 |
} else { |
| 718 |
eval { $marc = GetAuthority($record_number); }; |
672 |
eval { $marc = GetAuthority($record_number); }; |
|
Lines 918-928
Parameters:
Link Here
|
| 918 |
already exported the records |
872 |
already exported the records |
| 919 |
in a previous run. |
873 |
in a previous run. |
| 920 |
|
874 |
|
| 921 |
-noxml index from ISO MARC blob |
|
|
| 922 |
instead of MARC XML. This |
| 923 |
option is recommended only |
| 924 |
for advanced user. |
| 925 |
|
| 926 |
-nosanitize export biblio/authority records directly from DB marcxml |
875 |
-nosanitize export biblio/authority records directly from DB marcxml |
| 927 |
field without sanitizing records. It speed up |
876 |
field without sanitizing records. It speed up |
| 928 |
dump process but could fail if DB contains badly |
877 |
dump process but could fail if DB contains badly |
| 929 |
- |
|
|