Lines 40-47
use YAML::XS;
Link Here
|
40 |
|
40 |
|
41 |
use List::Util qw( sum0 ); |
41 |
use List::Util qw( sum0 ); |
42 |
use MARC::File::XML; |
42 |
use MARC::File::XML; |
43 |
use MIME::Base64 qw( encode_base64 ); |
43 |
use MIME::Base64 qw(encode_base64 decode_base64); |
44 |
use Encode qw( encode ); |
44 |
use Encode qw(encode decode); |
45 |
use Business::ISBN; |
45 |
use Business::ISBN; |
46 |
use Scalar::Util qw( looks_like_number ); |
46 |
use Scalar::Util qw( looks_like_number ); |
47 |
|
47 |
|
Lines 540-546
sub marc_records_to_documents {
Link Here
|
540 |
my $control_fields_rules = $rules->{control_fields}; |
540 |
my $control_fields_rules = $rules->{control_fields}; |
541 |
my $data_fields_rules = $rules->{data_fields}; |
541 |
my $data_fields_rules = $rules->{data_fields}; |
542 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
542 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
543 |
my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; |
|
|
544 |
|
543 |
|
545 |
my @record_documents; |
544 |
my @record_documents; |
546 |
|
545 |
|
Lines 692-729
sub marc_records_to_documents {
Link Here
|
692 |
} |
691 |
} |
693 |
} |
692 |
} |
694 |
} |
693 |
} |
|
|
694 |
my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); |
695 |
|
695 |
|
696 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
696 |
my ($encoded_record, $format) = $self->search_document_marc_record_encode( |
697 |
$record->encoding('UTF-8'); |
697 |
$record, |
698 |
if ($use_array) { |
698 |
$preferred_format, |
699 |
$record_document->{'marc_data_array'} = $self->_marc_to_array($record); |
699 |
$marcflavour |
700 |
$record_document->{'marc_format'} = 'ARRAY'; |
700 |
); |
|
|
701 |
|
702 |
if ($preferred_format eq 'ARRAY') { |
703 |
$record_document->{'marc_data_array'} = $encoded_record; |
701 |
} else { |
704 |
} else { |
702 |
my @warnings; |
705 |
$record_document->{'marc_data'} = $encoded_record; |
703 |
{ |
706 |
} |
704 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
707 |
$record_document->{'marc_format'} = $format; |
705 |
local $SIG{__WARN__} = sub { |
708 |
|
706 |
push @warnings, $_[0]; |
709 |
push @record_documents, $record_document; |
707 |
}; |
710 |
} |
708 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
711 |
return \@record_documents; |
709 |
} |
712 |
} |
710 |
if (@warnings) { |
713 |
|
711 |
# Suppress warnings if record length exceeded |
714 |
=head2 search_document_marc_record_encode($record, $format, $marcflavour) |
712 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
715 |
my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) |
713 |
foreach my $warning (@warnings) { |
716 |
|
714 |
carp $warning; |
717 |
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum |
715 |
} |
718 |
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. |
|
|
719 |
|
720 |
=over 4 |
721 |
|
722 |
=item C<$record> |
723 |
|
724 |
A MARC::Record object |
725 |
|
726 |
=item C<$marcflavour> |
727 |
|
728 |
The marcflavour to use |
729 |
|
730 |
=back |
731 |
|
732 |
=cut |
733 |
|
734 |
sub search_document_marc_record_encode { |
735 |
my ($self, $record, $format, $marcflavour) = @_; |
736 |
|
737 |
$record->encoding('UTF-8'); |
738 |
|
739 |
if ($format eq 'ARRAY') { |
740 |
return ($self->_marc_to_array($record), $format); |
741 |
} |
742 |
elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { |
743 |
my @warnings; |
744 |
my $marc_data; |
745 |
{ |
746 |
# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) |
747 |
local $SIG{__WARN__} = sub { |
748 |
push @warnings, $_[0]; |
749 |
}; |
750 |
$marc_data = $record->as_usmarc(); |
751 |
} |
752 |
if (@warnings) { |
753 |
# Suppress warnings if record length exceeded |
754 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
755 |
foreach my $warning (@warnings) { |
756 |
carp $warning; |
716 |
} |
757 |
} |
717 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
|
|
718 |
$record_document->{'marc_format'} = 'MARCXML'; |
719 |
} |
758 |
} |
720 |
else { |
759 |
return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); |
721 |
$record_document->{'marc_format'} = 'base64ISO2709'; |
760 |
} |
|
|
761 |
else { |
762 |
if ($format eq 'base64ISO2709') { |
763 |
$marc_data = encode_base64(encode('UTF-8', $marc_data)); |
722 |
} |
764 |
} |
|
|
765 |
return ($marc_data, $format); |
723 |
} |
766 |
} |
724 |
push @record_documents, $record_document; |
|
|
725 |
} |
767 |
} |
726 |
return \@record_documents; |
768 |
elsif ($format eq 'MARCXML') { |
|
|
769 |
return (MARC::File::XML::record($record, $marcflavour), $format); |
770 |
} |
771 |
else { |
772 |
# This should be unlikely to happen |
773 |
croak "Invalid marc record serialization format: $format"; |
774 |
} |
775 |
} |
776 |
|
777 |
=head2 search_document_marc_record_decode |
778 |
my $marc_record = $self->search_document_marc_record_decode(@result); |
779 |
|
780 |
Extract marc data from Elasticsearch result and decode to MARC::Record object |
781 |
|
782 |
=cut |
783 |
|
784 |
sub search_document_marc_record_decode { |
785 |
# Result is passed in as array, will get flattened |
786 |
# and first element will be $result |
787 |
my ($self, $result) = @_; |
788 |
if ($result->{marc_format} eq 'base64ISO2709') { |
789 |
return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); |
790 |
} |
791 |
elsif ($result->{marc_format} eq 'MARCXML') { |
792 |
return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); |
793 |
} |
794 |
elsif ($result->{marc_format} eq 'ARRAY') { |
795 |
return $self->_array_to_marc($result->{marc_data_array}); |
796 |
} |
797 |
else { |
798 |
Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); |
799 |
} |
800 |
} |
801 |
|
802 |
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) |
803 |
|
804 |
$records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) |
805 |
|
806 |
Return marc encoded records from ElasticSearch search result documents. The return value |
807 |
C<$marc_records> is a hashref with encoded records keyed by MARC format. |
808 |
|
809 |
=over 4 |
810 |
|
811 |
=item C<$docs> |
812 |
|
813 |
An arrayref of Elasticsearch search documents |
814 |
|
815 |
=item C<$preferred_format> |
816 |
|
817 |
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum |
818 |
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> |
819 |
is set to 'ISO2709'. |
820 |
|
821 |
=back |
822 |
|
823 |
=cut |
824 |
|
825 |
sub search_document_marc_records_encode_from_docs { |
826 |
|
827 |
my ($self, $docs, $preferred_format) = @_; |
828 |
|
829 |
my %encoded_records = ( |
830 |
'ISO2709' => [], |
831 |
'MARCXML' => [] |
832 |
); |
833 |
|
834 |
unless (exists $encoded_records{$preferred_format}) { |
835 |
croak "Invalid preferred format: $preferred_format"; |
836 |
} |
837 |
|
838 |
for my $es_record (@{$docs}) { |
839 |
# Special optimized cases |
840 |
my $marc_data; |
841 |
my $resulting_format = $preferred_format; |
842 |
if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { |
843 |
$marc_data = $es_record->{_source}{marc_data}; |
844 |
} |
845 |
elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { |
846 |
# Stored as UTF-8 encoded binary in index, so needs to be decoded |
847 |
$marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); |
848 |
} |
849 |
else { |
850 |
my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); |
851 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
852 |
($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); |
853 |
} |
854 |
push @{$encoded_records{$resulting_format}}, $marc_data; |
855 |
} |
856 |
if (@{$encoded_records{'ISO2709'}}) { |
857 |
$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); |
858 |
} |
859 |
else { |
860 |
delete $encoded_records{'ISO2709'}; |
861 |
} |
862 |
|
863 |
if (@{$encoded_records{'MARCXML'}}) { |
864 |
$encoded_records{'MARCXML'} = join("\n", |
865 |
MARC::File::XML::header(), |
866 |
join("\n", @{$encoded_records{'MARCXML'}}), |
867 |
MARC::File::XML::footer() |
868 |
); |
869 |
} |
870 |
else { |
871 |
delete $encoded_records{'MARCXML'}; |
872 |
} |
873 |
|
874 |
return \%encoded_records; |
727 |
} |
875 |
} |
728 |
|
876 |
|
729 |
=head2 _marc_to_array($record) |
877 |
=head2 _marc_to_array($record) |
Lines 797-814
sub _array_to_marc {
Link Here
|
797 |
$record->leader($data->{leader}); |
945 |
$record->leader($data->{leader}); |
798 |
for my $field (@{$data->{fields}}) { |
946 |
for my $field (@{$data->{fields}}) { |
799 |
my $tag = (keys %{$field})[0]; |
947 |
my $tag = (keys %{$field})[0]; |
800 |
$field = $field->{$tag}; |
948 |
my $field_data = $field->{$tag}; |
801 |
my $marc_field; |
949 |
my $marc_field; |
802 |
if (ref($field) eq 'HASH') { |
950 |
if (ref($field_data) eq 'HASH') { |
803 |
my @subfields; |
951 |
my @subfields; |
804 |
foreach my $subfield (@{$field->{subfields}}) { |
952 |
foreach my $subfield (@{$field_data->{subfields}}) { |
805 |
my $code = (keys %{$subfield})[0]; |
953 |
my $code = (keys %{$subfield})[0]; |
806 |
push @subfields, $code; |
954 |
push @subfields, $code; |
807 |
push @subfields, $subfield->{$code}; |
955 |
push @subfields, $subfield->{$code}; |
808 |
} |
956 |
} |
809 |
$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); |
957 |
$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); |
810 |
} else { |
958 |
} else { |
811 |
$marc_field = MARC::Field->new($tag, $field) |
959 |
$marc_field = MARC::Field->new($tag, $field_data) |
812 |
} |
960 |
} |
813 |
$record->append_fields($marc_field); |
961 |
$record->append_fields($marc_field); |
814 |
} |
962 |
} |