|
Lines 42-49
use YAML::XS;
Link Here
|
| 42 |
|
42 |
|
| 43 |
use List::Util qw( sum0 ); |
43 |
use List::Util qw( sum0 ); |
| 44 |
use MARC::File::XML; |
44 |
use MARC::File::XML; |
| 45 |
use MIME::Base64 qw( encode_base64 ); |
45 |
use MIME::Base64 qw(encode_base64 decode_base64); |
| 46 |
use Encode qw( encode ); |
46 |
use Encode qw(encode decode); |
| 47 |
use Business::ISBN; |
47 |
use Business::ISBN; |
| 48 |
use Scalar::Util qw( looks_like_number ); |
48 |
use Scalar::Util qw( looks_like_number ); |
| 49 |
|
49 |
|
|
Lines 542-548
sub marc_records_to_documents {
Link Here
|
| 542 |
my $control_fields_rules = $rules->{control_fields}; |
542 |
my $control_fields_rules = $rules->{control_fields}; |
| 543 |
my $data_fields_rules = $rules->{data_fields}; |
543 |
my $data_fields_rules = $rules->{data_fields}; |
| 544 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
544 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 545 |
my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; |
|
|
| 546 |
|
545 |
|
| 547 |
my @record_documents; |
546 |
my @record_documents; |
| 548 |
|
547 |
|
|
Lines 742-776
sub marc_records_to_documents {
Link Here
|
| 742 |
} |
741 |
} |
| 743 |
} |
742 |
} |
| 744 |
} |
743 |
} |
|
|
744 |
my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); |
| 745 |
|
745 |
|
| 746 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
746 |
my ($encoded_record, $format) = $self->search_document_marc_record_encode( |
| 747 |
$record->encoding('UTF-8'); |
747 |
$record, |
| 748 |
if ($use_array) { |
748 |
$preferred_format, |
| 749 |
$record_document->{'marc_data_array'} = $self->_marc_to_array($record); |
749 |
$marcflavour |
| 750 |
$record_document->{'marc_format'} = 'ARRAY'; |
750 |
); |
|
|
751 |
|
| 752 |
if ($preferred_format eq 'ARRAY') { |
| 753 |
$record_document->{'marc_data_array'} = $encoded_record; |
| 751 |
} else { |
754 |
} else { |
| 752 |
my @warnings; |
755 |
$record_document->{'marc_data'} = $encoded_record; |
| 753 |
{ |
|
|
| 754 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
| 755 |
local $SIG{__WARN__} = sub { |
| 756 |
push @warnings, $_[0]; |
| 757 |
}; |
| 758 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
| 759 |
} |
| 760 |
if (@warnings) { |
| 761 |
# Suppress warnings if record length exceeded |
| 762 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 763 |
foreach my $warning (@warnings) { |
| 764 |
carp $warning; |
| 765 |
} |
| 766 |
} |
| 767 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
| 768 |
$record_document->{'marc_format'} = 'MARCXML'; |
| 769 |
} |
| 770 |
else { |
| 771 |
$record_document->{'marc_format'} = 'base64ISO2709'; |
| 772 |
} |
| 773 |
} |
756 |
} |
|
|
757 |
$record_document->{'marc_format'} = $format; |
| 758 |
|
| 774 |
|
759 |
|
| 775 |
# Check if there is at least one available item |
760 |
# Check if there is at least one available item |
| 776 |
if ($self->index eq $BIBLIOS_INDEX) { |
761 |
if ($self->index eq $BIBLIOS_INDEX) { |
|
Lines 783-789
sub marc_records_to_documents {
Link Here
|
| 783 |
onloan => undef, |
768 |
onloan => undef, |
| 784 |
itemlost => 0, |
769 |
itemlost => 0, |
| 785 |
})->count; |
770 |
})->count; |
| 786 |
|
|
|
| 787 |
$record_document->{available} = $avail_items ? \1 : \0; |
771 |
$record_document->{available} = $avail_items ? \1 : \0; |
| 788 |
} |
772 |
} |
| 789 |
} |
773 |
} |
|
Lines 793-798
sub marc_records_to_documents {
Link Here
|
| 793 |
return \@record_documents; |
777 |
return \@record_documents; |
| 794 |
} |
778 |
} |
| 795 |
|
779 |
|
|
|
780 |
=head2 search_document_marc_record_encode($record, $format, $marcflavour) |
| 781 |
my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) |
| 782 |
|
| 783 |
Encode a MARC::Record to the preferred marc document record format. If record |
| 784 |
exceeds ISO2709 maximum size record size and C<$format> is set to |
| 785 |
'base64ISO2709' format will fallback to 'MARCXML' instead. |
| 786 |
|
| 787 |
=over 4 |
| 788 |
|
| 789 |
=item C<$record> |
| 790 |
|
| 791 |
A MARC::Record object |
| 792 |
|
| 793 |
=item C<$marcflavour> |
| 794 |
|
| 795 |
The marcflavour to use |
| 796 |
|
| 797 |
=back |
| 798 |
|
| 799 |
=cut |
| 800 |
|
| 801 |
sub search_document_marc_record_encode { |
| 802 |
my ($self, $record, $format, $marcflavour) = @_; |
| 803 |
|
| 804 |
$record->encoding('UTF-8'); |
| 805 |
|
| 806 |
if ($format eq 'ARRAY') { |
| 807 |
return ($self->_marc_to_array($record), $format); |
| 808 |
} |
| 809 |
elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { |
| 810 |
my @warnings; |
| 811 |
my $marc_data; |
| 812 |
{ |
| 813 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
| 814 |
local $SIG{__WARN__} = sub { |
| 815 |
push @warnings, $_[0]; |
| 816 |
}; |
| 817 |
$marc_data = $record->as_usmarc(); |
| 818 |
} |
| 819 |
if (@warnings) { |
| 820 |
# Suppress warnings if record length exceeded |
| 821 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 822 |
foreach my $warning (@warnings) { |
| 823 |
carp $warning; |
| 824 |
} |
| 825 |
} |
| 826 |
return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); |
| 827 |
} |
| 828 |
else { |
| 829 |
if ($format eq 'base64ISO2709') { |
| 830 |
$marc_data = encode_base64(encode('UTF-8', $marc_data)); |
| 831 |
} |
| 832 |
return ($marc_data, $format); |
| 833 |
} |
| 834 |
} |
| 835 |
elsif ($format eq 'MARCXML') { |
| 836 |
return (MARC::File::XML::record($record, $marcflavour), $format); |
| 837 |
} |
| 838 |
else { |
| 839 |
# This should be unlikely to happen |
| 840 |
croak "Invalid marc record serialization format: $format"; |
| 841 |
} |
| 842 |
} |
| 843 |
|
| 844 |
=head2 search_document_marc_record_decode |
| 845 |
my $marc_record = $self->search_document_marc_record_decode(@result); |
| 846 |
|
| 847 |
Extract marc data from Elasticsearch result and decode to MARC::Record object |
| 848 |
|
| 849 |
=cut |
| 850 |
|
| 851 |
sub search_document_marc_record_decode { |
| 852 |
# Result is passed in as array, will get flattened |
| 853 |
# and first element will be $result |
| 854 |
my ($self, $result) = @_; |
| 855 |
if ($result->{marc_format} eq 'base64ISO2709') { |
| 856 |
return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); |
| 857 |
} |
| 858 |
elsif ($result->{marc_format} eq 'MARCXML') { |
| 859 |
return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); |
| 860 |
} |
| 861 |
elsif ($result->{marc_format} eq 'ARRAY') { |
| 862 |
return $self->_array_to_marc($result->{marc_data_array}); |
| 863 |
} |
| 864 |
else { |
| 865 |
Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); |
| 866 |
} |
| 867 |
} |
| 868 |
|
| 869 |
=head2 search_documents_encode($docs, $preferred_format) |
| 870 |
|
| 871 |
$records_data = $self->search_documents_encode($docs, $preferred_format) |
| 872 |
|
| 873 |
Return marc encoded records from ElasticSearch search result documents. The return value |
| 874 |
C<$marc_records> is a hashref with encoded records keyed by MARC format. |
| 875 |
|
| 876 |
=over 4 |
| 877 |
|
| 878 |
=item C<$docs> |
| 879 |
|
| 880 |
An arrayref of Elasticsearch search documents |
| 881 |
|
| 882 |
=item C<$preferred_format> |
| 883 |
|
| 884 |
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum |
| 885 |
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> |
| 886 |
is set to 'ISO2709'. |
| 887 |
|
| 888 |
=back |
| 889 |
|
| 890 |
=cut |
| 891 |
|
| 892 |
sub search_documents_encode { |
| 893 |
|
| 894 |
my ($self, $docs, $preferred_format) = @_; |
| 895 |
|
| 896 |
my %encoded_records = ( |
| 897 |
'ISO2709' => [], |
| 898 |
'MARCXML' => [] |
| 899 |
); |
| 900 |
|
| 901 |
unless (exists $encoded_records{$preferred_format}) { |
| 902 |
croak "Invalid preferred format: $preferred_format"; |
| 903 |
} |
| 904 |
|
| 905 |
for my $es_record (@{$docs}) { |
| 906 |
# Special optimized cases |
| 907 |
my $marc_data; |
| 908 |
my $resulting_format = $preferred_format; |
| 909 |
if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { |
| 910 |
$marc_data = $es_record->{_source}{marc_data}; |
| 911 |
} |
| 912 |
elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { |
| 913 |
$marc_data = decode_base64($es_record->{_source}->{marc_data}); |
| 914 |
} |
| 915 |
else { |
| 916 |
my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); |
| 917 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 918 |
($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); |
| 919 |
} |
| 920 |
push @{$encoded_records{$resulting_format}}, $marc_data; |
| 921 |
} |
| 922 |
if (@{$encoded_records{'ISO2709'}}) { |
| 923 |
$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); |
| 924 |
} |
| 925 |
else { |
| 926 |
delete $encoded_records{'ISO2709'}; |
| 927 |
} |
| 928 |
|
| 929 |
if (@{$encoded_records{'MARCXML'}}) { |
| 930 |
$encoded_records{'MARCXML'} = encode( |
| 931 |
'UTF-8', |
| 932 |
join( |
| 933 |
"\n", |
| 934 |
MARC::File::XML::header(), |
| 935 |
join("\n", @{$encoded_records{'MARCXML'}}), |
| 936 |
MARC::File::XML::footer() |
| 937 |
) |
| 938 |
); |
| 939 |
} |
| 940 |
else { |
| 941 |
delete $encoded_records{'MARCXML'}; |
| 942 |
} |
| 943 |
|
| 944 |
return \%encoded_records; |
| 945 |
} |
| 946 |
|
| 947 |
=head2 search_result_export_custom_formats() |
| 948 |
|
| 949 |
$custom_formats = $self->search_result_export_custom_formats() |
| 950 |
|
| 951 |
Return user defined custom search result export formats. |
| 952 |
|
| 953 |
=cut |
| 954 |
|
| 955 |
sub search_result_export_custom_formats { |
| 956 |
my $export_custom_formats_pref = C4::Context->yaml_preference('SearchResultMARCExportCustomFormats') || []; |
| 957 |
my $custom_export_formats = {}; |
| 958 |
|
| 959 |
if (ref $export_custom_formats_pref eq 'ARRAY') { |
| 960 |
for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) { |
| 961 |
# TODO: Perhaps validate on save or trow error here instead of just |
| 962 |
# ignoring invalid formats |
| 963 |
my $format = $export_custom_formats_pref->[$i]; |
| 964 |
if ( |
| 965 |
ref $format->{fields} eq 'ARRAY' && |
| 966 |
@{$format->{fields}} && |
| 967 |
$format->{name} |
| 968 |
) { |
| 969 |
$format->{multiple} = 'ignore' unless exists $format->{multiple}; |
| 970 |
$custom_export_formats->{"custom_$i"} = $format; |
| 971 |
} |
| 972 |
} |
| 973 |
} |
| 974 |
return $custom_export_formats; |
| 975 |
} |
| 976 |
|
| 977 |
=head2 search_documents_custom_format_encode($docs, $custom_format) |
| 978 |
|
| 979 |
$records_data = $self->search_documents_custom_format_encode($docs, $custom_format) |
| 980 |
|
| 981 |
Return encoded records from ElasticSearch search result documents using a |
| 982 |
custom format defined in the "SearchResultMARCExportCustomFormats" syspref. |
| 983 |
Returns the encoded records. |
| 984 |
|
| 985 |
=over 4 |
| 986 |
|
| 987 |
=item C<$docs> |
| 988 |
|
| 989 |
An arrayref of Elasticsearch search documents |
| 990 |
|
| 991 |
=item C<$format> |
| 992 |
|
| 993 |
A hashref with the custom format definition. |
| 994 |
|
| 995 |
=back |
| 996 |
|
| 997 |
=cut |
| 998 |
|
| 999 |
sub search_documents_custom_format_encode { |
| 1000 |
my ($self, $docs, $format) = @_; |
| 1001 |
|
| 1002 |
my $result; |
| 1003 |
|
| 1004 |
my $doc_get_fields = sub { |
| 1005 |
my ($doc, $fields) = @_; |
| 1006 |
my @row; |
| 1007 |
foreach my $field (@{$fields}) { |
| 1008 |
my $values = $doc->{_source}->{$field}; |
| 1009 |
push @row, ref $values eq 'ARRAY' ? $values : ['']; |
| 1010 |
} |
| 1011 |
return \@row; |
| 1012 |
}; |
| 1013 |
|
| 1014 |
my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs}; |
| 1015 |
|
| 1016 |
if($format->{multiple} eq 'ignore') { |
| 1017 |
for (my $i = 0; $i < @rows; ++$i) { |
| 1018 |
$rows[$i] = [map { $_->[0] } @{$rows[$i]}]; |
| 1019 |
} |
| 1020 |
} |
| 1021 |
elsif($format->{multiple} eq 'newline') { |
| 1022 |
if (@{$format->{fields}} == 1) { |
| 1023 |
@rows = map { [join("\n", @{$_->[0]})] } @rows; |
| 1024 |
} |
| 1025 |
else { |
| 1026 |
croak "'newline' is only valid for single field export formats"; |
| 1027 |
} |
| 1028 |
} |
| 1029 |
elsif($format->{multiple} eq 'join') { |
| 1030 |
for (my $i = 0; $i < @rows; ++$i) { |
| 1031 |
# Escape separator |
| 1032 |
for (my $j = 0; $j < @{$rows[$i]}; ++$j) { |
| 1033 |
for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) { |
| 1034 |
$rows[$i][$j][$k] =~ s/\|/\\|/g; |
| 1035 |
} |
| 1036 |
} |
| 1037 |
# Separate multiple values with "|" |
| 1038 |
$rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}]; |
| 1039 |
} |
| 1040 |
} |
| 1041 |
else { |
| 1042 |
croak "Invalid 'multiple' option: " . $format->{multiple}; |
| 1043 |
} |
| 1044 |
if (@{$format->{fields}} == 1) { |
| 1045 |
@rows = grep { $_ ne '' } map { $_->[0] } @rows; |
| 1046 |
} |
| 1047 |
else { |
| 1048 |
# Encode CSV |
| 1049 |
for (my $i = 0; $i < @rows; ++$i) { |
| 1050 |
# Escape quotes |
| 1051 |
for (my $j = 0; $j < @{$rows[$i]}; ++$j) { |
| 1052 |
$rows[$i][$j] =~ s/"/""/g; |
| 1053 |
} |
| 1054 |
$rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]}); |
| 1055 |
} |
| 1056 |
} |
| 1057 |
|
| 1058 |
return encode('UTF-8', join("\n", @rows)); |
| 1059 |
} |
| 1060 |
|
| 796 |
=head2 _marc_to_array($record) |
1061 |
=head2 _marc_to_array($record) |
| 797 |
|
1062 |
|
| 798 |
my @fields = _marc_to_array($record) |
1063 |
my @fields = _marc_to_array($record) |
|
Lines 864-881
sub _array_to_marc {
Link Here
|
| 864 |
$record->leader($data->{leader}); |
1129 |
$record->leader($data->{leader}); |
| 865 |
for my $field (@{$data->{fields}}) { |
1130 |
for my $field (@{$data->{fields}}) { |
| 866 |
my $tag = (keys %{$field})[0]; |
1131 |
my $tag = (keys %{$field})[0]; |
| 867 |
$field = $field->{$tag}; |
1132 |
my $field_data = $field->{$tag}; |
| 868 |
my $marc_field; |
1133 |
my $marc_field; |
| 869 |
if (ref($field) eq 'HASH') { |
1134 |
if (ref($field_data) eq 'HASH') { |
| 870 |
my @subfields; |
1135 |
my @subfields; |
| 871 |
foreach my $subfield (@{$field->{subfields}}) { |
1136 |
foreach my $subfield (@{$field_data->{subfields}}) { |
| 872 |
my $code = (keys %{$subfield})[0]; |
1137 |
my $code = (keys %{$subfield})[0]; |
| 873 |
push @subfields, $code; |
1138 |
push @subfields, $code; |
| 874 |
push @subfields, $subfield->{$code}; |
1139 |
push @subfields, $subfield->{$code}; |
| 875 |
} |
1140 |
} |
| 876 |
$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); |
1141 |
$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); |
| 877 |
} else { |
1142 |
} else { |
| 878 |
$marc_field = MARC::Field->new($tag, $field) |
1143 |
$marc_field = MARC::Field->new($tag, $field_data) |
| 879 |
} |
1144 |
} |
| 880 |
$record->append_fields($marc_field); |
1145 |
$record->append_fields($marc_field); |
| 881 |
} |
1146 |
} |