|
Lines 41-48
use YAML::Syck;
Link Here
|
| 41 |
|
41 |
|
| 42 |
use List::Util qw( sum0 reduce ); |
42 |
use List::Util qw( sum0 reduce ); |
| 43 |
use MARC::File::XML; |
43 |
use MARC::File::XML; |
| 44 |
use MIME::Base64; |
44 |
use MIME::Base64 qw(encode_base64 decode_base64); |
| 45 |
use Encode qw(encode); |
45 |
use Encode qw(encode decode); |
| 46 |
use Business::ISBN; |
46 |
use Business::ISBN; |
| 47 |
use Scalar::Util qw(looks_like_number); |
47 |
use Scalar::Util qw(looks_like_number); |
| 48 |
|
48 |
|
|
Lines 524-530
sub marc_records_to_documents {
Link Here
|
| 524 |
my $control_fields_rules = $rules->{control_fields}; |
524 |
my $control_fields_rules = $rules->{control_fields}; |
| 525 |
my $data_fields_rules = $rules->{data_fields}; |
525 |
my $data_fields_rules = $rules->{data_fields}; |
| 526 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
526 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 527 |
my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; |
|
|
| 528 |
|
527 |
|
| 529 |
my @record_documents; |
528 |
my @record_documents; |
| 530 |
|
529 |
|
|
Lines 679-716
sub marc_records_to_documents {
Link Here
|
| 679 |
} |
678 |
} |
| 680 |
} |
679 |
} |
| 681 |
} |
680 |
} |
|
|
681 |
my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); |
| 682 |
|
682 |
|
| 683 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
683 |
my ($encoded_record, $format) = $self->search_document_marc_record_encode( |
| 684 |
$record->encoding('UTF-8'); |
684 |
$record, |
| 685 |
if ($use_array) { |
685 |
$preferred_format, |
| 686 |
$record_document->{'marc_data_array'} = $self->_marc_to_array($record); |
686 |
$marcflavour |
| 687 |
$record_document->{'marc_format'} = 'ARRAY'; |
687 |
); |
|
|
688 |
|
| 689 |
if ($preferred_format eq 'ARRAY') { |
| 690 |
$record_document->{'marc_data_array'} = $encoded_record; |
| 688 |
} else { |
691 |
} else { |
| 689 |
my @warnings; |
692 |
$record_document->{'marc_data'} = $encoded_record; |
| 690 |
{ |
693 |
} |
| 691 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
694 |
$record_document->{'marc_format'} = $format; |
| 692 |
local $SIG{__WARN__} = sub { |
695 |
|
| 693 |
push @warnings, $_[0]; |
696 |
push @record_documents, $record_document; |
| 694 |
}; |
697 |
} |
| 695 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
698 |
return \@record_documents; |
| 696 |
} |
699 |
} |
| 697 |
if (@warnings) { |
700 |
|
| 698 |
# Suppress warnings if record length exceeded |
701 |
=head2 search_document_marc_record_encode($record, $format, $marcflavour) |
| 699 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
702 |
my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) |
| 700 |
foreach my $warning (@warnings) { |
703 |
|
| 701 |
carp $warning; |
704 |
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum |
| 702 |
} |
705 |
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. |
|
|
706 |
|
| 707 |
=over 4 |
| 708 |
|
| 709 |
=item C<$record> |
| 710 |
|
| 711 |
A MARC::Record object |
| 712 |
|
| 713 |
=item C<$marcflavour> |
| 714 |
|
| 715 |
The marcflavour to use |
| 716 |
|
| 717 |
=back |
| 718 |
|
| 719 |
=cut |
| 720 |
|
| 721 |
sub search_document_marc_record_encode { |
| 722 |
my ($self, $record, $format, $marcflavour) = @_; |
| 723 |
|
| 724 |
$record->encoding('UTF-8'); |
| 725 |
|
| 726 |
if ($format eq 'ARRAY') { |
| 727 |
return ($self->_marc_to_array($record), $format); |
| 728 |
} |
| 729 |
elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { |
| 730 |
my @warnings; |
| 731 |
my $marc_data; |
| 732 |
{ |
| 733 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
| 734 |
local $SIG{__WARN__} = sub { |
| 735 |
push @warnings, $_[0]; |
| 736 |
}; |
| 737 |
$marc_data = $record->as_usmarc(); |
| 738 |
} |
| 739 |
if (@warnings) { |
| 740 |
# Suppress warnings if record length exceeded |
| 741 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 742 |
foreach my $warning (@warnings) { |
| 743 |
carp $warning; |
| 703 |
} |
744 |
} |
| 704 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
|
|
| 705 |
$record_document->{'marc_format'} = 'MARCXML'; |
| 706 |
} |
745 |
} |
| 707 |
else { |
746 |
return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); |
| 708 |
$record_document->{'marc_format'} = 'base64ISO2709'; |
747 |
} |
|
|
748 |
else { |
| 749 |
if ($format eq 'base64ISO2709') { |
| 750 |
$marc_data = encode_base64(encode('UTF-8', $marc_data)); |
| 709 |
} |
751 |
} |
|
|
752 |
return ($marc_data, $format); |
| 710 |
} |
753 |
} |
| 711 |
push @record_documents, $record_document; |
|
|
| 712 |
} |
754 |
} |
| 713 |
return \@record_documents; |
755 |
elsif ($format eq 'MARCXML') { |
|
|
756 |
return (MARC::File::XML::record($record, $marcflavour), $format); |
| 757 |
} |
| 758 |
else { |
| 759 |
# This should be unlikely to happen |
| 760 |
croak "Invalid marc record serialization format: $format"; |
| 761 |
} |
| 762 |
} |
| 763 |
|
| 764 |
=head2 search_document_marc_record_decode |
| 765 |
my $marc_record = $self->search_document_marc_record_decode(@result); |
| 766 |
|
| 767 |
Extract marc data from Elasticsearch result and decode to MARC::Record object |
| 768 |
|
| 769 |
=cut |
| 770 |
|
| 771 |
sub search_document_marc_record_decode { |
| 772 |
# Result is passed in as array, will get flattened |
| 773 |
# and first element will be $result |
| 774 |
my ($self, $result) = @_; |
| 775 |
if ($result->{marc_format} eq 'base64ISO2709') { |
| 776 |
return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); |
| 777 |
} |
| 778 |
elsif ($result->{marc_format} eq 'MARCXML') { |
| 779 |
return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); |
| 780 |
} |
| 781 |
elsif ($result->{marc_format} eq 'ARRAY') { |
| 782 |
return $self->_array_to_marc($result->{marc_data_array}); |
| 783 |
} |
| 784 |
else { |
| 785 |
Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); |
| 786 |
} |
| 787 |
} |
| 788 |
|
| 789 |
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 790 |
|
| 791 |
$records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 792 |
|
| 793 |
Return marc encoded records from ElasticSearch search result documents. The return value |
| 794 |
C<$marc_records> is a hashref with encoded records keyed by MARC format. |
| 795 |
|
| 796 |
=over 4 |
| 797 |
|
| 798 |
=item C<$docs> |
| 799 |
|
| 800 |
An arrayref of Elasticsearch search documents |
| 801 |
|
| 802 |
=item C<$preferred_format> |
| 803 |
|
| 804 |
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum |
| 805 |
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> |
| 806 |
is set to 'ISO2709'. |
| 807 |
|
| 808 |
=back |
| 809 |
|
| 810 |
=cut |
| 811 |
|
| 812 |
sub search_document_marc_records_encode_from_docs { |
| 813 |
|
| 814 |
my ($self, $docs, $preferred_format) = @_; |
| 815 |
|
| 816 |
my %encoded_records = ( |
| 817 |
'ISO2709' => [], |
| 818 |
'MARCXML' => [] |
| 819 |
); |
| 820 |
|
| 821 |
unless (exists $encoded_records{$preferred_format}) { |
| 822 |
croak "Invalid preferred format: $preferred_format"; |
| 823 |
} |
| 824 |
|
| 825 |
for my $es_record (@{$docs}) { |
| 826 |
# Special optimized cases |
| 827 |
my $marc_data; |
| 828 |
my $resulting_format = $preferred_format; |
| 829 |
if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { |
| 830 |
$marc_data = $es_record->{_source}{marc_data}; |
| 831 |
} |
| 832 |
elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { |
| 833 |
# Stored as UTF-8 encoded binary in index, so needs to be decoded |
| 834 |
$marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); |
| 835 |
} |
| 836 |
else { |
| 837 |
my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); |
| 838 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 839 |
($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); |
| 840 |
} |
| 841 |
push @{$encoded_records{$resulting_format}}, $marc_data; |
| 842 |
} |
| 843 |
if (@{$encoded_records{'ISO2709'}}) { |
| 844 |
$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); |
| 845 |
} |
| 846 |
else { |
| 847 |
delete $encoded_records{'ISO2709'}; |
| 848 |
} |
| 849 |
|
| 850 |
if (@{$encoded_records{'MARCXML'}}) { |
| 851 |
$encoded_records{'MARCXML'} = join("\n", |
| 852 |
MARC::File::XML::header(), |
| 853 |
join("\n", @{$encoded_records{'MARCXML'}}), |
| 854 |
MARC::File::XML::footer() |
| 855 |
); |
| 856 |
} |
| 857 |
else { |
| 858 |
delete $encoded_records{'MARCXML'}; |
| 859 |
} |
| 860 |
|
| 861 |
return \%encoded_records; |
| 714 |
} |
862 |
} |
| 715 |
|
863 |
|
| 716 |
=head2 _marc_to_array($record) |
864 |
=head2 _marc_to_array($record) |
|
Lines 784-801
sub _array_to_marc {
Link Here
|
| 784 |
$record->leader($data->{leader}); |
932 |
$record->leader($data->{leader}); |
| 785 |
for my $field (@{$data->{fields}}) { |
933 |
for my $field (@{$data->{fields}}) { |
| 786 |
my $tag = (keys %{$field})[0]; |
934 |
my $tag = (keys %{$field})[0]; |
| 787 |
$field = $field->{$tag}; |
935 |
my $field_data = $field->{$tag}; |
| 788 |
my $marc_field; |
936 |
my $marc_field; |
| 789 |
if (ref($field) eq 'HASH') { |
937 |
if (ref($field_data) eq 'HASH') { |
| 790 |
my @subfields; |
938 |
my @subfields; |
| 791 |
foreach my $subfield (@{$field->{subfields}}) { |
939 |
foreach my $subfield (@{$field_data->{subfields}}) { |
| 792 |
my $code = (keys %{$subfield})[0]; |
940 |
my $code = (keys %{$subfield})[0]; |
| 793 |
push @subfields, $code; |
941 |
push @subfields, $code; |
| 794 |
push @subfields, $subfield->{$code}; |
942 |
push @subfields, $subfield->{$code}; |
| 795 |
} |
943 |
} |
| 796 |
$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); |
944 |
$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); |
| 797 |
} else { |
945 |
} else { |
| 798 |
$marc_field = MARC::Field->new($tag, $field) |
946 |
$marc_field = MARC::Field->new($tag, $field_data) |
| 799 |
} |
947 |
} |
| 800 |
$record->append_fields($marc_field); |
948 |
$record->append_fields($marc_field); |
| 801 |
} |
949 |
} |