|
Lines 40-47
use YAML::XS;
Link Here
|
| 40 |
|
40 |
|
| 41 |
use List::Util qw( sum0 ); |
41 |
use List::Util qw( sum0 ); |
| 42 |
use MARC::File::XML; |
42 |
use MARC::File::XML; |
| 43 |
use MIME::Base64 qw( encode_base64 ); |
43 |
use MIME::Base64 qw(encode_base64 decode_base64); |
| 44 |
use Encode qw( encode ); |
44 |
use Encode qw(encode decode); |
| 45 |
use Business::ISBN; |
45 |
use Business::ISBN; |
| 46 |
use Scalar::Util qw( looks_like_number ); |
46 |
use Scalar::Util qw( looks_like_number ); |
| 47 |
|
47 |
|
|
Lines 540-546
sub marc_records_to_documents {
Link Here
|
| 540 |
my $control_fields_rules = $rules->{control_fields}; |
540 |
my $control_fields_rules = $rules->{control_fields}; |
| 541 |
my $data_fields_rules = $rules->{data_fields}; |
541 |
my $data_fields_rules = $rules->{data_fields}; |
| 542 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
542 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 543 |
my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; |
|
|
| 544 |
|
543 |
|
| 545 |
my @record_documents; |
544 |
my @record_documents; |
| 546 |
|
545 |
|
|
Lines 692-729
sub marc_records_to_documents {
Link Here
|
| 692 |
} |
691 |
} |
| 693 |
} |
692 |
} |
| 694 |
} |
693 |
} |
|
|
694 |
my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); |
| 695 |
|
695 |
|
| 696 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
696 |
my ($encoded_record, $format) = $self->search_document_marc_record_encode( |
| 697 |
$record->encoding('UTF-8'); |
697 |
$record, |
| 698 |
if ($use_array) { |
698 |
$preferred_format, |
| 699 |
$record_document->{'marc_data_array'} = $self->_marc_to_array($record); |
699 |
$marcflavour |
| 700 |
$record_document->{'marc_format'} = 'ARRAY'; |
700 |
); |
|
|
701 |
|
| 702 |
if ($preferred_format eq 'ARRAY') { |
| 703 |
$record_document->{'marc_data_array'} = $encoded_record; |
| 701 |
} else { |
704 |
} else { |
| 702 |
my @warnings; |
705 |
$record_document->{'marc_data'} = $encoded_record; |
| 703 |
{ |
706 |
} |
| 704 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
707 |
$record_document->{'marc_format'} = $format; |
| 705 |
local $SIG{__WARN__} = sub { |
708 |
|
| 706 |
push @warnings, $_[0]; |
709 |
push @record_documents, $record_document; |
| 707 |
}; |
710 |
} |
| 708 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
711 |
return \@record_documents; |
| 709 |
} |
712 |
} |
| 710 |
if (@warnings) { |
713 |
|
| 711 |
# Suppress warnings if record length exceeded |
714 |
=head2 search_document_marc_record_encode($record, $format, $marcflavour) |
| 712 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
715 |
my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) |
| 713 |
foreach my $warning (@warnings) { |
716 |
|
| 714 |
carp $warning; |
717 |
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum |
| 715 |
} |
718 |
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. |
|
|
719 |
|
| 720 |
=over 4 |
| 721 |
|
| 722 |
=item C<$record> |
| 723 |
|
| 724 |
A MARC::Record object |
| 725 |
|
| 726 |
=item C<$marcflavour> |
| 727 |
|
| 728 |
The marcflavour to use |
| 729 |
|
| 730 |
=back |
| 731 |
|
| 732 |
=cut |
| 733 |
|
| 734 |
sub search_document_marc_record_encode { |
| 735 |
my ($self, $record, $format, $marcflavour) = @_; |
| 736 |
|
| 737 |
$record->encoding('UTF-8'); |
| 738 |
|
| 739 |
if ($format eq 'ARRAY') { |
| 740 |
return ($self->_marc_to_array($record), $format); |
| 741 |
} |
| 742 |
elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { |
| 743 |
my @warnings; |
| 744 |
my $marc_data; |
| 745 |
{ |
| 746 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
| 747 |
local $SIG{__WARN__} = sub { |
| 748 |
push @warnings, $_[0]; |
| 749 |
}; |
| 750 |
$marc_data = $record->as_usmarc(); |
| 751 |
} |
| 752 |
if (@warnings) { |
| 753 |
# Suppress warnings if record length exceeded |
| 754 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 755 |
foreach my $warning (@warnings) { |
| 756 |
carp $warning; |
| 716 |
} |
757 |
} |
| 717 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
|
|
| 718 |
$record_document->{'marc_format'} = 'MARCXML'; |
| 719 |
} |
758 |
} |
| 720 |
else { |
759 |
return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); |
| 721 |
$record_document->{'marc_format'} = 'base64ISO2709'; |
760 |
} |
|
|
761 |
else { |
| 762 |
if ($format eq 'base64ISO2709') { |
| 763 |
$marc_data = encode_base64(encode('UTF-8', $marc_data)); |
| 722 |
} |
764 |
} |
|
|
765 |
return ($marc_data, $format); |
| 723 |
} |
766 |
} |
| 724 |
push @record_documents, $record_document; |
|
|
| 725 |
} |
767 |
} |
| 726 |
return \@record_documents; |
768 |
elsif ($format eq 'MARCXML') { |
|
|
769 |
return (MARC::File::XML::record($record, $marcflavour), $format); |
| 770 |
} |
| 771 |
else { |
| 772 |
# This should be unlikely to happen |
| 773 |
croak "Invalid marc record serialization format: $format"; |
| 774 |
} |
| 775 |
} |
| 776 |
|
| 777 |
=head2 search_document_marc_record_decode |
| 778 |
my $marc_record = $self->search_document_marc_record_decode(@result); |
| 779 |
|
| 780 |
Extract marc data from Elasticsearch result and decode to MARC::Record object |
| 781 |
|
| 782 |
=cut |
| 783 |
|
| 784 |
sub search_document_marc_record_decode { |
| 785 |
# Result is passed in as array, will get flattened |
| 786 |
# and first element will be $result |
| 787 |
my ($self, $result) = @_; |
| 788 |
if ($result->{marc_format} eq 'base64ISO2709') { |
| 789 |
return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); |
| 790 |
} |
| 791 |
elsif ($result->{marc_format} eq 'MARCXML') { |
| 792 |
return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); |
| 793 |
} |
| 794 |
elsif ($result->{marc_format} eq 'ARRAY') { |
| 795 |
return $self->_array_to_marc($result->{marc_data_array}); |
| 796 |
} |
| 797 |
else { |
| 798 |
Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); |
| 799 |
} |
| 800 |
} |
| 801 |
|
| 802 |
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 803 |
|
| 804 |
$records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 805 |
|
| 806 |
Return marc encoded records from ElasticSearch search result documents. The return value |
| 807 |
C<$marc_records> is a hashref with encoded records keyed by MARC format. |
| 808 |
|
| 809 |
=over 4 |
| 810 |
|
| 811 |
=item C<$docs> |
| 812 |
|
| 813 |
An arrayref of Elasticsearch search documents |
| 814 |
|
| 815 |
=item C<$preferred_format> |
| 816 |
|
| 817 |
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum |
| 818 |
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> |
| 819 |
is set to 'ISO2709'. |
| 820 |
|
| 821 |
=back |
| 822 |
|
| 823 |
=cut |
| 824 |
|
| 825 |
sub search_document_marc_records_encode_from_docs { |
| 826 |
|
| 827 |
my ($self, $docs, $preferred_format) = @_; |
| 828 |
|
| 829 |
my %encoded_records = ( |
| 830 |
'ISO2709' => [], |
| 831 |
'MARCXML' => [] |
| 832 |
); |
| 833 |
|
| 834 |
unless (exists $encoded_records{$preferred_format}) { |
| 835 |
croak "Invalid preferred format: $preferred_format"; |
| 836 |
} |
| 837 |
|
| 838 |
for my $es_record (@{$docs}) { |
| 839 |
# Special optimized cases |
| 840 |
my $marc_data; |
| 841 |
my $resulting_format = $preferred_format; |
| 842 |
if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { |
| 843 |
$marc_data = $es_record->{_source}{marc_data}; |
| 844 |
} |
| 845 |
elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { |
| 846 |
# Stored as UTF-8 encoded binary in index, so needs to be decoded |
| 847 |
$marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); |
| 848 |
} |
| 849 |
else { |
| 850 |
my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); |
| 851 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 852 |
($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); |
| 853 |
} |
| 854 |
push @{$encoded_records{$resulting_format}}, $marc_data; |
| 855 |
} |
| 856 |
if (@{$encoded_records{'ISO2709'}}) { |
| 857 |
$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); |
| 858 |
} |
| 859 |
else { |
| 860 |
delete $encoded_records{'ISO2709'}; |
| 861 |
} |
| 862 |
|
| 863 |
if (@{$encoded_records{'MARCXML'}}) { |
| 864 |
$encoded_records{'MARCXML'} = join("\n", |
| 865 |
MARC::File::XML::header(), |
| 866 |
join("\n", @{$encoded_records{'MARCXML'}}), |
| 867 |
MARC::File::XML::footer() |
| 868 |
); |
| 869 |
} |
| 870 |
else { |
| 871 |
delete $encoded_records{'MARCXML'}; |
| 872 |
} |
| 873 |
|
| 874 |
return \%encoded_records; |
| 727 |
} |
875 |
} |
| 728 |
|
876 |
|
| 729 |
=head2 _marc_to_array($record) |
877 |
=head2 _marc_to_array($record) |
|
Lines 797-814
sub _array_to_marc {
Link Here
|
| 797 |
$record->leader($data->{leader}); |
945 |
$record->leader($data->{leader}); |
| 798 |
for my $field (@{$data->{fields}}) { |
946 |
for my $field (@{$data->{fields}}) { |
| 799 |
my $tag = (keys %{$field})[0]; |
947 |
my $tag = (keys %{$field})[0]; |
| 800 |
$field = $field->{$tag}; |
948 |
my $field_data = $field->{$tag}; |
| 801 |
my $marc_field; |
949 |
my $marc_field; |
| 802 |
if (ref($field) eq 'HASH') { |
950 |
if (ref($field_data) eq 'HASH') { |
| 803 |
my @subfields; |
951 |
my @subfields; |
| 804 |
foreach my $subfield (@{$field->{subfields}}) { |
952 |
foreach my $subfield (@{$field_data->{subfields}}) { |
| 805 |
my $code = (keys %{$subfield})[0]; |
953 |
my $code = (keys %{$subfield})[0]; |
| 806 |
push @subfields, $code; |
954 |
push @subfields, $code; |
| 807 |
push @subfields, $subfield->{$code}; |
955 |
push @subfields, $subfield->{$code}; |
| 808 |
} |
956 |
} |
| 809 |
$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); |
957 |
$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); |
| 810 |
} else { |
958 |
} else { |
| 811 |
$marc_field = MARC::Field->new($tag, $field) |
959 |
$marc_field = MARC::Field->new($tag, $field_data) |
| 812 |
} |
960 |
} |
| 813 |
$record->append_fields($marc_field); |
961 |
$record->append_fields($marc_field); |
| 814 |
} |
962 |
} |