|
Lines 41-48
use YAML::XS;
Link Here
|
| 41 |
|
41 |
|
| 42 |
use List::Util qw( sum0 ); |
42 |
use List::Util qw( sum0 ); |
| 43 |
use MARC::File::XML; |
43 |
use MARC::File::XML; |
| 44 |
use MIME::Base64 qw( encode_base64 ); |
44 |
use MIME::Base64 qw(encode_base64 decode_base64); |
| 45 |
use Encode qw( encode ); |
45 |
use Encode qw(encode decode); |
| 46 |
use Business::ISBN; |
46 |
use Business::ISBN; |
| 47 |
use Scalar::Util qw( looks_like_number ); |
47 |
use Scalar::Util qw( looks_like_number ); |
| 48 |
|
48 |
|
|
Lines 541-547
sub marc_records_to_documents {
Link Here
|
| 541 |
my $control_fields_rules = $rules->{control_fields}; |
541 |
my $control_fields_rules = $rules->{control_fields}; |
| 542 |
my $data_fields_rules = $rules->{data_fields}; |
542 |
my $data_fields_rules = $rules->{data_fields}; |
| 543 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
543 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 544 |
my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; |
|
|
| 545 |
|
544 |
|
| 546 |
my @record_documents; |
545 |
my @record_documents; |
| 547 |
|
546 |
|
|
Lines 741-778
sub marc_records_to_documents {
Link Here
|
| 741 |
} |
740 |
} |
| 742 |
} |
741 |
} |
| 743 |
} |
742 |
} |
|
|
743 |
my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); |
| 744 |
|
744 |
|
| 745 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
745 |
my ($encoded_record, $format) = $self->search_document_marc_record_encode( |
| 746 |
$record->encoding('UTF-8'); |
746 |
$record, |
| 747 |
if ($use_array) { |
747 |
$preferred_format, |
| 748 |
$record_document->{'marc_data_array'} = $self->_marc_to_array($record); |
748 |
$marcflavour |
| 749 |
$record_document->{'marc_format'} = 'ARRAY'; |
749 |
); |
|
|
750 |
|
| 751 |
if ($preferred_format eq 'ARRAY') { |
| 752 |
$record_document->{'marc_data_array'} = $encoded_record; |
| 750 |
} else { |
753 |
} else { |
| 751 |
my @warnings; |
754 |
$record_document->{'marc_data'} = $encoded_record; |
| 752 |
{ |
755 |
} |
| 753 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
756 |
$record_document->{'marc_format'} = $format; |
| 754 |
local $SIG{__WARN__} = sub { |
757 |
|
| 755 |
push @warnings, $_[0]; |
758 |
push @record_documents, $record_document; |
| 756 |
}; |
759 |
} |
| 757 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
760 |
return \@record_documents; |
| 758 |
} |
761 |
} |
| 759 |
if (@warnings) { |
762 |
|
| 760 |
# Suppress warnings if record length exceeded |
763 |
=head2 search_document_marc_record_encode($record, $format, $marcflavour) |
| 761 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
764 |
my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) |
| 762 |
foreach my $warning (@warnings) { |
765 |
|
| 763 |
carp $warning; |
766 |
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum |
| 764 |
} |
767 |
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. |
|
|
768 |
|
| 769 |
=over 4 |
| 770 |
|
| 771 |
=item C<$record> |
| 772 |
|
| 773 |
A MARC::Record object |
| 774 |
|
| 775 |
=item C<$marcflavour> |
| 776 |
|
| 777 |
The marcflavour to use |
| 778 |
|
| 779 |
=back |
| 780 |
|
| 781 |
=cut |
| 782 |
|
| 783 |
sub search_document_marc_record_encode { |
| 784 |
my ($self, $record, $format, $marcflavour) = @_; |
| 785 |
|
| 786 |
$record->encoding('UTF-8'); |
| 787 |
|
| 788 |
if ($format eq 'ARRAY') { |
| 789 |
return ($self->_marc_to_array($record), $format); |
| 790 |
} |
| 791 |
elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { |
| 792 |
my @warnings; |
| 793 |
my $marc_data; |
| 794 |
{ |
| 795 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
| 796 |
local $SIG{__WARN__} = sub { |
| 797 |
push @warnings, $_[0]; |
| 798 |
}; |
| 799 |
$marc_data = $record->as_usmarc(); |
| 800 |
} |
| 801 |
if (@warnings) { |
| 802 |
# Suppress warnings if record length exceeded |
| 803 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 804 |
foreach my $warning (@warnings) { |
| 805 |
carp $warning; |
| 765 |
} |
806 |
} |
| 766 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
|
|
| 767 |
$record_document->{'marc_format'} = 'MARCXML'; |
| 768 |
} |
807 |
} |
| 769 |
else { |
808 |
return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); |
| 770 |
$record_document->{'marc_format'} = 'base64ISO2709'; |
809 |
} |
|
|
810 |
else { |
| 811 |
if ($format eq 'base64ISO2709') { |
| 812 |
$marc_data = encode_base64(encode('UTF-8', $marc_data)); |
| 771 |
} |
813 |
} |
|
|
814 |
return ($marc_data, $format); |
| 772 |
} |
815 |
} |
| 773 |
push @record_documents, $record_document; |
|
|
| 774 |
} |
816 |
} |
| 775 |
return \@record_documents; |
817 |
elsif ($format eq 'MARCXML') { |
|
|
818 |
return (MARC::File::XML::record($record, $marcflavour), $format); |
| 819 |
} |
| 820 |
else { |
| 821 |
# This should be unlikely to happen |
| 822 |
croak "Invalid marc record serialization format: $format"; |
| 823 |
} |
| 824 |
} |
| 825 |
|
| 826 |
=head2 search_document_marc_record_decode |
| 827 |
my $marc_record = $self->search_document_marc_record_decode(@result); |
| 828 |
|
| 829 |
Extract marc data from Elasticsearch result and decode to MARC::Record object |
| 830 |
|
| 831 |
=cut |
| 832 |
|
| 833 |
sub search_document_marc_record_decode { |
| 834 |
# Result is passed in as array, will get flattened |
| 835 |
# and first element will be $result |
| 836 |
my ($self, $result) = @_; |
| 837 |
if ($result->{marc_format} eq 'base64ISO2709') { |
| 838 |
return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); |
| 839 |
} |
| 840 |
elsif ($result->{marc_format} eq 'MARCXML') { |
| 841 |
return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); |
| 842 |
} |
| 843 |
elsif ($result->{marc_format} eq 'ARRAY') { |
| 844 |
return $self->_array_to_marc($result->{marc_data_array}); |
| 845 |
} |
| 846 |
else { |
| 847 |
Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); |
| 848 |
} |
| 849 |
} |
| 850 |
|
| 851 |
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 852 |
|
| 853 |
$records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) |
| 854 |
|
| 855 |
Return marc encoded records from ElasticSearch search result documents. The return value |
| 856 |
C<$marc_records> is a hashref with encoded records keyed by MARC format. |
| 857 |
|
| 858 |
=over 4 |
| 859 |
|
| 860 |
=item C<$docs> |
| 861 |
|
| 862 |
An arrayref of Elasticsearch search documents |
| 863 |
|
| 864 |
=item C<$preferred_format> |
| 865 |
|
| 866 |
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum |
| 867 |
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> |
| 868 |
is set to 'ISO2709'. |
| 869 |
|
| 870 |
=back |
| 871 |
|
| 872 |
=cut |
| 873 |
|
| 874 |
sub search_document_marc_records_encode_from_docs { |
| 875 |
|
| 876 |
my ($self, $docs, $preferred_format) = @_; |
| 877 |
|
| 878 |
my %encoded_records = ( |
| 879 |
'ISO2709' => [], |
| 880 |
'MARCXML' => [] |
| 881 |
); |
| 882 |
|
| 883 |
unless (exists $encoded_records{$preferred_format}) { |
| 884 |
croak "Invalid preferred format: $preferred_format"; |
| 885 |
} |
| 886 |
|
| 887 |
for my $es_record (@{$docs}) { |
| 888 |
# Special optimized cases |
| 889 |
my $marc_data; |
| 890 |
my $resulting_format = $preferred_format; |
| 891 |
if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { |
| 892 |
$marc_data = $es_record->{_source}{marc_data}; |
| 893 |
} |
| 894 |
elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { |
| 895 |
# Stored as UTF-8 encoded binary in index, so needs to be decoded |
| 896 |
$marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); |
| 897 |
} |
| 898 |
else { |
| 899 |
my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); |
| 900 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 901 |
($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); |
| 902 |
} |
| 903 |
push @{$encoded_records{$resulting_format}}, $marc_data; |
| 904 |
} |
| 905 |
if (@{$encoded_records{'ISO2709'}}) { |
| 906 |
$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); |
| 907 |
} |
| 908 |
else { |
| 909 |
delete $encoded_records{'ISO2709'}; |
| 910 |
} |
| 911 |
|
| 912 |
if (@{$encoded_records{'MARCXML'}}) { |
| 913 |
$encoded_records{'MARCXML'} = join("\n", |
| 914 |
MARC::File::XML::header(), |
| 915 |
join("\n", @{$encoded_records{'MARCXML'}}), |
| 916 |
MARC::File::XML::footer() |
| 917 |
); |
| 918 |
} |
| 919 |
else { |
| 920 |
delete $encoded_records{'MARCXML'}; |
| 921 |
} |
| 922 |
|
| 923 |
return \%encoded_records; |
| 776 |
} |
924 |
} |
| 777 |
|
925 |
|
| 778 |
=head2 _marc_to_array($record) |
926 |
=head2 _marc_to_array($record) |
|
Lines 846-863
sub _array_to_marc {
Link Here
|
| 846 |
$record->leader($data->{leader}); |
994 |
$record->leader($data->{leader}); |
| 847 |
for my $field (@{$data->{fields}}) { |
995 |
for my $field (@{$data->{fields}}) { |
| 848 |
my $tag = (keys %{$field})[0]; |
996 |
my $tag = (keys %{$field})[0]; |
| 849 |
$field = $field->{$tag}; |
997 |
my $field_data = $field->{$tag}; |
| 850 |
my $marc_field; |
998 |
my $marc_field; |
| 851 |
if (ref($field) eq 'HASH') { |
999 |
if (ref($field_data) eq 'HASH') { |
| 852 |
my @subfields; |
1000 |
my @subfields; |
| 853 |
foreach my $subfield (@{$field->{subfields}}) { |
1001 |
foreach my $subfield (@{$field_data->{subfields}}) { |
| 854 |
my $code = (keys %{$subfield})[0]; |
1002 |
my $code = (keys %{$subfield})[0]; |
| 855 |
push @subfields, $code; |
1003 |
push @subfields, $code; |
| 856 |
push @subfields, $subfield->{$code}; |
1004 |
push @subfields, $subfield->{$code}; |
| 857 |
} |
1005 |
} |
| 858 |
$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); |
1006 |
$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); |
| 859 |
} else { |
1007 |
} else { |
| 860 |
$marc_field = MARC::Field->new($tag, $field) |
1008 |
$marc_field = MARC::Field->new($tag, $field_data) |
| 861 |
} |
1009 |
} |
| 862 |
$record->append_fields($marc_field); |
1010 |
$record->append_fields($marc_field); |
| 863 |
} |
1011 |
} |