View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/BackgroundJob.pm (+1 lines)
Lines 426-431 sub core_types_to_classes { Link Here
426
        stage_marc_for_import               => 'Koha::BackgroundJob::StageMARCForImport',
426
        stage_marc_for_import               => 'Koha::BackgroundJob::StageMARCForImport',
427
        marc_import_commit_batch            => 'Koha::BackgroundJob::MARCImportCommitBatch',
427
        marc_import_commit_batch            => 'Koha::BackgroundJob::MARCImportCommitBatch',
428
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
428
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
429
        search_result_export                => 'Koha::BackgroundJob::SearchResultExport',
429
    };
430
    };
430
}
431
}
431
432
(-)a/Koha/BackgroundJob/SearchResultExport.pm (+185 lines)
Line 0 Link Here
1
package Koha::BackgroundJob::SearchResultExport;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
use Try::Tiny;
20
use Koha::SearchEngine::Search;
21
22
use File::Spec;
23
use File::Path qw(mkpath);
24
use Koha::Email;
25
use Koha::UploadedFiles;
26
use POSIX qw(strftime);
27
use Digest::MD5 qw(md5_hex);
28
use Carp qw(croak);
29
30
use base 'Koha::BackgroundJob';
31
32
=head1 NAME
33
34
Koha::BackgroundJob::SearchResultExport - Export data from search result
35
36
This is a subclass of Koha::BackgroundJob.
37
38
=head1 API
39
40
=head2 Class methods
41
42
=head3 job_type
43
44
Define the job type of this job: stage_marc_for_import
45
46
=cut
47
48
sub job_type {
49
    return 'search_result_export';
50
}
51
52
=head3 process
53
54
Perform the export of search records.
55
56
=cut
57
58
sub process {
59
    my ( $self, $args ) = @_;
60
61
    $self->start;
62
63
    my $data = $self->decoded_data;
64
    my $borrowernumber = $data->{borrowernumber};
65
    my $elasticsearch_query = $args->{elasticsearch_query};
66
    my $preferred_format = $args->{preferred_format};
67
    my $searcher = Koha::SearchEngine::Search->new({
68
        index => $Koha::SearchEngine::BIBLIOS_INDEX
69
    });
70
    my $elasticsearch = $searcher->get_elasticsearch();
71
72
    my $results = eval {
73
        $elasticsearch->search(
74
            index => $searcher->index_name,
75
            scroll => '1m', #TODO: Syspref for scroll time limit?
76
            size => 1000,  #TODO: Syspref for batch size?
77
            body => $elasticsearch_query
78
        );
79
    };
80
    my @errors;
81
    push @errors, $@ if $@;
82
83
    my @docs;
84
    my $encoded_results;
85
    my %export_links;
86
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
87
88
    if (!@errors) {
89
        my $scroll_id = $results->{_scroll_id};
90
        while (@{$results->{hits}->{hits}}) {
91
            push @docs, @{$results->{hits}->{hits}};
92
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
93
            $results = $elasticsearch->scroll(
94
                scroll => '1m',
95
                scroll_id => $scroll_id
96
            );
97
        }
98
99
        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
100
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
101
        }
102
        else {
103
            $encoded_results->{$preferred_format->{name}} =
104
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
105
        }
106
107
        my %format_extensions = (
108
            'ISO2709' => '.mrc',
109
            'MARCXML' => '.xml',
110
        );
111
112
        my $upload_dir = Koha::UploadedFile->permanent_directory;
113
114
        while (my ($format, $data) = each %{$encoded_results}) {
115
            my $hash = md5_hex($data);
116
            my $category = "search_marc_export";
117
            my $time = strftime "%Y%m%d_%H%M", localtime time;
118
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
119
            my $filename = $category . '_' . $time . $ext;
120
            my $file_dir = File::Spec->catfile($upload_dir, $category);
121
            if ( !-d $file_dir) {
122
                unless(mkpath $file_dir) {
123
                    push @errors, "Failed to create $file_dir";
124
                    next;
125
                }
126
            }
127
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
128
129
            my $fh = IO::File->new($filepath, "w");
130
131
            if ($fh) {
132
                $fh->binmode;
133
                print $fh $data;
134
                $fh->close;
135
136
                my $size = -s $filepath;
137
                my $file = Koha::UploadedFile->new({
138
                        hashvalue => $hash,
139
                        filename  => $filename,
140
                        dir       => $category,
141
                        filesize  => $size,
142
                        owner     => $borrowernumber,
143
                        uploadcategorycode => 'search_marc_export',
144
                        public    => 0,
145
                        permanent => 1,
146
                    })->store;
147
                my $id = $file->_result()->get_column('id');
148
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
149
            }
150
            else {
151
                push @errors, "Failed to write \"$filepath\"";
152
            }
153
        }
154
    }
155
    my $report = {
156
        export_links => \%export_links,
157
        total => scalar @docs,
158
        errors => \@errors,
159
        query_string => $query_string,
160
    };
161
    $data->{report}   = $report;
162
    if (@errors) {
163
        $self->set({ progress => 0, status => 'failed' })->store;
164
    }
165
    else {
166
        $self->finish($data);
167
    }
168
}
169
170
=head3 enqueue
171
172
Enqueue the new job
173
174
=cut
175
176
sub enqueue {
177
    my ( $self, $args) = @_;
178
    $self->SUPER::enqueue({
179
        job_size => $args->{size},
180
        job_args => $args,
181
        job_queue => 'long_tasks',
182
    });
183
}
184
185
1;
(-)a/Koha/SearchEngine/Elasticsearch.pm (-35 / +300 lines)
Lines 42-49 use YAML::XS; Link Here
42
42
43
use List::Util qw( sum0 );
43
use List::Util qw( sum0 );
44
use MARC::File::XML;
44
use MARC::File::XML;
45
use MIME::Base64 qw( encode_base64 );
45
use MIME::Base64 qw(encode_base64 decode_base64);
46
use Encode qw( encode );
46
use Encode qw(encode decode);
47
use Business::ISBN;
47
use Business::ISBN;
48
use Scalar::Util qw( looks_like_number );
48
use Scalar::Util qw( looks_like_number );
49
49
Lines 542-548 sub marc_records_to_documents { Link Here
542
    my $control_fields_rules = $rules->{control_fields};
542
    my $control_fields_rules = $rules->{control_fields};
543
    my $data_fields_rules = $rules->{data_fields};
543
    my $data_fields_rules = $rules->{data_fields};
544
    my $marcflavour = lc C4::Context->preference('marcflavour');
544
    my $marcflavour = lc C4::Context->preference('marcflavour');
545
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
546
545
547
    my @record_documents;
546
    my @record_documents;
548
547
Lines 742-776 sub marc_records_to_documents { Link Here
742
                }
741
                }
743
            }
742
            }
744
        }
743
        }
744
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
745
745
746
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
746
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
747
        $record->encoding('UTF-8');
747
            $record,
748
        if ($use_array) {
748
            $preferred_format,
749
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
749
            $marcflavour
750
            $record_document->{'marc_format'} = 'ARRAY';
750
        );
751
752
        if ($preferred_format eq 'ARRAY') {
753
            $record_document->{'marc_data_array'} = $encoded_record;
751
        } else {
754
        } else {
752
            my @warnings;
755
            $record_document->{'marc_data'} = $encoded_record;
753
            {
754
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
755
                local $SIG{__WARN__} = sub {
756
                    push @warnings, $_[0];
757
                };
758
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
759
            }
760
            if (@warnings) {
761
                # Suppress warnings if record length exceeded
762
                unless (substr($record->leader(), 0, 5) eq '99999') {
763
                    foreach my $warning (@warnings) {
764
                        carp $warning;
765
                    }
766
                }
767
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
768
                $record_document->{'marc_format'} = 'MARCXML';
769
            }
770
            else {
771
                $record_document->{'marc_format'} = 'base64ISO2709';
772
            }
773
        }
756
        }
757
        $record_document->{'marc_format'} = $format;
758
774
759
775
        # Check if there is at least one available item
760
        # Check if there is at least one available item
776
        if ($self->index eq $BIBLIOS_INDEX) {
761
        if ($self->index eq $BIBLIOS_INDEX) {
Lines 783-789 sub marc_records_to_documents { Link Here
783
                    onloan       => undef,
768
                    onloan       => undef,
784
                    itemlost     => 0,
769
                    itemlost     => 0,
785
                })->count;
770
                })->count;
786
787
                $record_document->{available} = $avail_items ? \1 : \0;
771
                $record_document->{available} = $avail_items ? \1 : \0;
788
            }
772
            }
789
        }
773
        }
Lines 793-798 sub marc_records_to_documents { Link Here
793
    return \@record_documents;
777
    return \@record_documents;
794
}
778
}
795
779
780
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
781
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
782
783
Encode a MARC::Record to the preferred marc document record format. If record
784
exceeds ISO2709 maximum size record size and C<$format> is set to
785
'base64ISO2709' format will fallback to 'MARCXML' instead.
786
787
=over 4
788
789
=item C<$record>
790
791
A MARC::Record object
792
793
=item C<$marcflavour>
794
795
The marcflavour to use
796
797
=back
798
799
=cut
800
801
sub search_document_marc_record_encode {
802
    my ($self, $record, $format, $marcflavour) = @_;
803
804
    $record->encoding('UTF-8');
805
806
    if ($format eq 'ARRAY') {
807
        return ($self->_marc_to_array($record), $format);
808
    }
809
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
810
        my @warnings;
811
        my $marc_data;
812
        {
813
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
814
            local $SIG{__WARN__} = sub {
815
                push @warnings, $_[0];
816
            };
817
            $marc_data = $record->as_usmarc();
818
        }
819
        if (@warnings) {
820
            # Suppress warnings if record length exceeded
821
            unless (substr($record->leader(), 0, 5) eq '99999') {
822
                foreach my $warning (@warnings) {
823
                    carp $warning;
824
                }
825
            }
826
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
827
        }
828
        else {
829
            if ($format eq 'base64ISO2709') {
830
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
831
            }
832
            return ($marc_data, $format);
833
        }
834
    }
835
    elsif ($format eq 'MARCXML') {
836
        return (MARC::File::XML::record($record, $marcflavour), $format);
837
    }
838
    else {
839
        # This should be unlikely to happen
840
        croak "Invalid marc record serialization format: $format";
841
    }
842
}
843
844
=head2 search_document_marc_record_decode
845
    my $marc_record = $self->search_document_marc_record_decode(@result);
846
847
Extract marc data from Elasticsearch result and decode to MARC::Record object
848
849
=cut
850
851
sub search_document_marc_record_decode {
852
    # Result is passed in as array, will get flattened
853
    # and first element will be $result
854
    my ($self, $result) = @_;
855
    if ($result->{marc_format} eq 'base64ISO2709') {
856
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
857
    }
858
    elsif ($result->{marc_format} eq 'MARCXML') {
859
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
860
    }
861
    elsif ($result->{marc_format} eq 'ARRAY') {
862
        return $self->_array_to_marc($result->{marc_data_array});
863
    }
864
    else {
865
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
866
    }
867
}
868
869
=head2 search_documents_encode($docs, $preferred_format)
870
871
    $records_data = $self->search_documents_encode($docs, $preferred_format)
872
873
Return marc encoded records from ElasticSearch search result documents. The return value
874
C<$marc_records> is a hashref with encoded records keyed by MARC format.
875
876
=over 4
877
878
=item C<$docs>
879
880
An arrayref of Elasticsearch search documents
881
882
=item C<$preferred_format>
883
884
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
885
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
886
is set to 'ISO2709'.
887
888
=back
889
890
=cut
891
892
sub search_documents_encode {
893
894
    my ($self, $docs, $preferred_format) = @_;
895
896
    my %encoded_records = (
897
        'ISO2709' => [],
898
        'MARCXML' => []
899
    );
900
901
    unless (exists $encoded_records{$preferred_format}) {
902
       croak "Invalid preferred format: $preferred_format";
903
    }
904
905
    for my $es_record (@{$docs}) {
906
        # Special optimized cases
907
        my $marc_data;
908
        my $resulting_format = $preferred_format;
909
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
910
            $marc_data = $es_record->{_source}{marc_data};
911
        }
912
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
913
            $marc_data = decode_base64($es_record->{_source}->{marc_data});
914
        }
915
        else {
916
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
917
            my $marcflavour = lc C4::Context->preference('marcflavour');
918
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
919
        }
920
        push @{$encoded_records{$resulting_format}}, $marc_data;
921
    }
922
    if (@{$encoded_records{'ISO2709'}}) {
923
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
924
    }
925
    else {
926
        delete $encoded_records{'ISO2709'};
927
    }
928
929
    if (@{$encoded_records{'MARCXML'}}) {
930
        $encoded_records{'MARCXML'} = encode(
931
            'UTF-8',
932
            join(
933
                "\n",
934
                MARC::File::XML::header(),
935
                join("\n", @{$encoded_records{'MARCXML'}}),
936
                MARC::File::XML::footer()
937
            )
938
        );
939
    }
940
    else {
941
        delete $encoded_records{'MARCXML'};
942
    }
943
944
    return \%encoded_records;
945
}
946
947
=head2 search_result_export_custom_formats()
948
949
    $custom_formats = $self->search_result_export_custom_formats()
950
951
Return user defined custom search result export formats.
952
953
=cut
954
955
sub search_result_export_custom_formats {
956
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
957
    my $custom_export_formats = {};
958
959
    if (ref $export_custom_formats_pref eq 'ARRAY') {
960
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
961
            # TODO: Perhaps validate on save or trow error here instead of just
962
            # ignoring invalid formats
963
            my $format = $export_custom_formats_pref->[$i];
964
            if (
965
                ref $format->{fields} eq 'ARRAY' &&
966
                @{$format->{fields}} &&
967
                $format->{name}
968
            ) {
969
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
970
                $custom_export_formats->{"custom_$i"} = $format;
971
            }
972
        }
973
    }
974
    return $custom_export_formats;
975
}
976
977
=head2 search_documents_custom_format_encode($docs, $custom_format)
978
979
    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
980
981
Return encoded records from ElasticSearch search result documents using a
982
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
983
Returns the encoded records.
984
985
=over 4
986
987
=item C<$docs>
988
989
An arrayref of Elasticsearch search documents
990
991
=item C<$format>
992
993
A hashref with the custom format definition.
994
995
=back
996
997
=cut
998
999
sub search_documents_custom_format_encode {
1000
    my ($self, $docs, $format) = @_;
1001
1002
    my $result;
1003
1004
    my $doc_get_fields = sub {
1005
        my ($doc, $fields) = @_;
1006
        my @row;
1007
        foreach my $field (@{$fields}) {
1008
            my $values = $doc->{_source}->{$field};
1009
            push @row, ref $values eq 'ARRAY' ? $values : [''];
1010
        }
1011
        return \@row;
1012
    };
1013
1014
    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1015
1016
    if($format->{multiple} eq 'ignore') {
1017
        for (my $i = 0; $i < @rows; ++$i) {
1018
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1019
        }
1020
    }
1021
    elsif($format->{multiple} eq 'newline') {
1022
        if (@{$format->{fields}} == 1) {
1023
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
1024
        }
1025
        else {
1026
            croak "'newline' is only valid for single field export formats";
1027
        }
1028
    }
1029
    elsif($format->{multiple} eq 'join') {
1030
        for (my $i = 0; $i < @rows; ++$i) {
1031
            # Escape separator
1032
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1033
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1034
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
1035
                }
1036
            }
1037
            # Separate multiple values with "|"
1038
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
1039
        }
1040
    }
1041
    else {
1042
        croak "Invalid 'multiple' option: " . $format->{multiple};
1043
    }
1044
    if (@{$format->{fields}} == 1) {
1045
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
1046
    }
1047
    else {
1048
        # Encode CSV
1049
        for (my $i = 0; $i < @rows; ++$i) {
1050
            # Escape quotes
1051
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1052
                $rows[$i][$j] =~ s/"/""/g;
1053
            }
1054
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1055
        }
1056
    }
1057
1058
    return encode('UTF-8', join("\n", @rows));
1059
}
1060
796
=head2 _marc_to_array($record)
1061
=head2 _marc_to_array($record)
797
1062
798
    my @fields = _marc_to_array($record)
1063
    my @fields = _marc_to_array($record)
Lines 864-881 sub _array_to_marc { Link Here
864
    $record->leader($data->{leader});
1129
    $record->leader($data->{leader});
865
    for my $field (@{$data->{fields}}) {
1130
    for my $field (@{$data->{fields}}) {
866
        my $tag = (keys %{$field})[0];
1131
        my $tag = (keys %{$field})[0];
867
        $field = $field->{$tag};
1132
        my $field_data = $field->{$tag};
868
        my $marc_field;
1133
        my $marc_field;
869
        if (ref($field) eq 'HASH') {
1134
        if (ref($field_data) eq 'HASH') {
870
            my @subfields;
1135
            my @subfields;
871
            foreach my $subfield (@{$field->{subfields}}) {
1136
            foreach my $subfield (@{$field_data->{subfields}}) {
872
                my $code = (keys %{$subfield})[0];
1137
                my $code = (keys %{$subfield})[0];
873
                push @subfields, $code;
1138
                push @subfields, $code;
874
                push @subfields, $subfield->{$code};
1139
                push @subfields, $subfield->{$code};
875
            }
1140
            }
876
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
1141
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
877
        } else {
1142
        } else {
878
            $marc_field = MARC::Field->new($tag, $field)
1143
            $marc_field = MARC::Field->new($tag, $field_data)
879
        }
1144
        }
880
        $record->append_fields($marc_field);
1145
        $record->append_fields($marc_field);
881
    }
1146
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-28 / +3 lines)
Lines 173-179 sub search_compat { Link Here
173
    my $index = $offset;
173
    my $index = $offset;
174
    my $hits = $results->{'hits'};
174
    my $hits = $results->{'hits'};
175
    foreach my $es_record (@{$hits->{'hits'}}) {
175
    foreach my $es_record (@{$hits->{'hits'}}) {
176
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
176
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
177
    }
177
    }
178
178
179
    # consumers of this expect a name-spaced result, we provide the default
179
    # consumers of this expect a name-spaced result, we provide the default
Lines 234-240 sub search_auth_compat { Link Here
234
            # it's not reproduced here yet.
234
            # it's not reproduced here yet.
235
            my $authtype           = $rs->single;
235
            my $authtype           = $rs->single;
236
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
236
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
237
            my $marc               = $self->decode_record_from_result($record);
237
            my $marc               = $self->search_document_marc_record_decode($record);
238
            my $mainentry          = $marc->field($auth_tag_to_report);
238
            my $mainentry          = $marc->field($auth_tag_to_report);
239
            my $reported_tag;
239
            my $reported_tag;
240
            if ($mainentry) {
240
            if ($mainentry) {
Lines 354-360 sub simple_search_compat { Link Here
354
    my @records;
354
    my @records;
355
    my $hits = $results->{'hits'};
355
    my $hits = $results->{'hits'};
356
    foreach my $es_record (@{$hits->{'hits'}}) {
356
    foreach my $es_record (@{$hits->{'hits'}}) {
357
        push @records, $self->decode_record_from_result($es_record->{'_source'});
357
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
358
    }
358
    }
359
    return (undef, \@records, $hits->{'total'});
359
    return (undef, \@records, $hits->{'total'});
360
}
360
}
Lines 374-404 sub extract_biblionumber { Link Here
374
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
374
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
375
}
375
}
376
376
377
=head2 decode_record_from_result
378
    my $marc_record = $self->decode_record_from_result(@result);
379
380
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
381
382
=cut
383
384
sub decode_record_from_result {
385
    # Result is passed in as array, will get flattened
386
    # and first element will be $result
387
    my ( $self, $result ) = @_;
388
    if ($result->{marc_format} eq 'base64ISO2709') {
389
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
390
    }
391
    elsif ($result->{marc_format} eq 'MARCXML') {
392
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
393
    }
394
    elsif ($result->{marc_format} eq 'ARRAY') {
395
        return $self->_array_to_marc($result->{marc_data_array});
396
    }
397
    else {
398
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
399
    }
400
}
401
402
=head2 max_result_window
377
=head2 max_result_window
403
378
404
Returns the maximum number of results that can be fetched
379
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+46 lines)
Lines 149-154 use C4::Koha qw( getitemtypeimagelocation GetAuthorisedValues ); Link Here
149
use URI::Escape;
149
use URI::Escape;
150
use POSIX qw(ceil floor);
150
use POSIX qw(ceil floor);
151
use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
151
use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
152
use Koha::BackgroundJob::SearchResultExport;
152
153
153
use Koha::ItemTypes;
154
use Koha::ItemTypes;
154
use Koha::Library::Groups;
155
use Koha::Library::Groups;
Lines 161-166 use Koha::SearchFilters; Link Here
161
162
162
use URI::Escape;
163
use URI::Escape;
163
use JSON qw( decode_json encode_json );
164
use JSON qw( decode_json encode_json );
165
use Carp qw(croak);
164
166
165
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
167
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
166
# create a new CGI object
168
# create a new CGI object
Lines 725-730 for (my $i=0;$i<@servers;$i++) { Link Here
725
} #/end of the for loop
727
} #/end of the for loop
726
#$template->param(FEDERATED_RESULTS => \@results_array);
728
#$template->param(FEDERATED_RESULTS => \@results_array);
727
729
730
my $patron = Koha::Patrons->find( $borrowernumber );
731
my $export_enabled =
732
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
733
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
734
    $patron && $patron->has_permission({ tools => 'export_catalog' });
735
736
$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
737
738
if ($export_enabled) {
739
740
741
    my $export = $cgi->param('export');
742
    my $preferred_format = $cgi->param('export_format');
743
    my $custom_export_formats = $searcher->search_result_export_custom_formats;
744
745
    $template->param(custom_export_formats => $custom_export_formats);
746
747
    # TODO: Need to handle $hits = 0?
748
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
749
750
    if ($export && $preferred_format && $hits) {
751
        unless (
752
            $preferred_format eq 'ISO2709' ||
753
            $preferred_format eq 'MARCXML'
754
        ) {
755
            if (!exists $custom_export_formats->{$preferred_format}) {
756
                croak "Invalid export format: $preferred_format";
757
            }
758
            else {
759
                $preferred_format = $custom_export_formats->{$preferred_format};
760
            }
761
        }
762
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
763
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
764
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
765
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
766
            size => $size,
767
            preferred_format => $preferred_format,
768
            elasticsearch_query => \%export_query
769
        });
770
        $template->param(export_job_id => $export_job_id);
771
    }
772
}
773
728
my $gotonumber = $cgi->param('gotoNumber');
774
my $gotonumber = $cgi->param('gotoNumber');
729
if ( $gotonumber && ( $gotonumber eq 'last' || $gotonumber eq 'first' ) ) {
775
if ( $gotonumber && ( $gotonumber eq 'last' || $gotonumber eq 'first' ) ) {
730
    $template->{'VARS'}->{'gotoNumber'} = $gotonumber;
776
    $template->{'VARS'}->{'gotoNumber'} = $gotonumber;
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+23 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
11
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
12
13
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
14
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
15
16
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
17
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
18
19
        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
20
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
21
        say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
22
    },
23
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +4 lines)
Lines 204-210 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
204
('EdifactLSQ', 'location', 'location|ccode', 'Map EDI sequence code (GIR+LSQ) to Koha Item field', 'Choice'),
204
('EdifactLSQ', 'location', 'location|ccode', 'Map EDI sequence code (GIR+LSQ) to Koha Item field', 'Choice'),
205
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
205
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
206
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
206
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
207
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
207
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
208
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
208
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
209
('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
209
('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
210
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
210
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
Lines 220-225 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
220
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
220
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
221
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
221
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
222
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
222
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
223
('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
223
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
224
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
224
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
225
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
225
('ERMModule', '0', NULL, 'Enable the e-resource management module', 'YesNo'),
226
('ERMModule', '0', NULL, 'Enable the e-resource management module', 'YesNo'),
Lines 640-645 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
640
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
641
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
641
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
642
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
642
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
643
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
644
('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
645
('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer'),
643
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
646
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
644
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
647
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
645
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
648
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
Line 0 Link Here
1
[% USE Koha %]
2
3
[% BLOCK report %]
4
    [% SET report = job.report %]
5
    [% IF report %]
6
        [% IF job.status == 'finished' %]
7
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
8
            <ul>
9
              <li>[% report.total | html %] records exported</li>
10
              [% IF report.export_links.keys.size > 1 %]
11
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
12
              [% END %]
13
              [% FOREACH format IN report.export_links.keys.sort %]
14
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
15
              [% END %]
16
            </ul>
17
        [% ELSE %]
18
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
19
            <ul>
20
            [% FOREACH error in report.errors %]
21
              <li>[% error | html %]</li>
22
            [% END %]
23
            </ul>
24
        [% END %]
25
    [% END %]
26
[% END %]
27
28
[% BLOCK detail %]
29
[% END %]
30
31
[% BLOCK js %]
32
[% END %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
Lines 233-238 Link Here
233
                '_id': 'marc_import_revert_batch',
233
                '_id': 'marc_import_revert_batch',
234
                '_str': _("Revert import MARC records")
234
                '_str': _("Revert import MARC records")
235
            },
235
            },
236
            {
237
                '_id': 'search_result_export',
238
                '_str': _("Search result export")
239
            },
236
        ];
240
        ];
237
241
238
        function get_job_type (job_type) {
242
        function get_job_type (job_type) {
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
Lines 474-482 Administration: Link Here
474
        -
474
        -
475
            - "Elasticsearch MARC format: "
475
            - "Elasticsearch MARC format: "
476
            - pref: ElasticsearchMARCFormat
476
            - pref: ElasticsearchMARCFormat
477
              default: "ISO2709"
477
              default: "base64ISO2709"
478
              choices:
478
              choices:
479
                "ISO2709": "ISO2709 (exchange format)"
479
                "base64ISO2709": "ISO2709 (exchange format)"
480
                "ARRAY": "Searchable array"
480
                "ARRAY": "Searchable array"
481
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
481
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
482
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
482
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
Lines 138-143 Searching: Link Here
138
                  1: use
138
                  1: use
139
                  0: "don't use"
139
                  0: "don't use"
140
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
140
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
141
        -
142
            - pref: EnableElasticsearchSearchResultExport
143
              type: boolean
144
              default: yes
145
              choices:
146
                  1: Enable
147
                  0: Disable
148
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
149
        -
150
            - pref: ElasticsearchSearchResultExportCustomFormats
151
              type: textarea
152
              syntax: text/x-yaml
153
              class: code
154
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
155
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
156
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
157
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
158
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
159
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
160
            - <p>The behavior when handling fields with multiple values.</p>
161
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
162
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
163
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
164
            - 'Example:</br>'
165
            - '- name: Biblionumbers<br />'
166
            - '&nbsp;&nbsp;fields: [local-number]<br />'
167
            - '&nbsp;&nbsp;multiple: ignore<br />'
168
            - '- name: Title and author<br />'
169
            - '&nbsp;&nbsp;fields: [title, author]<br />'
170
            - '&nbsp;&nbsp;multiple: join<br /><br />'
171
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
172
        -
173
            - Limit export from search results to a maximum of
174
            - pref: ElasticsearchSearchResultExportLimit
175
              class: integer
176
            - search result items (Elasticsearch only).<br /><br />
177
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
141
    Results display:
178
    Results display:
142
        -
179
        -
143
            - pref: numSearchResultsDropdown
180
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+51 lines)
Lines 329-334 Link Here
329
                                </div> <!-- /.btn-group -->
329
                                </div> <!-- /.btn-group -->
330
                            [% END %]
330
                            [% END %]
331
331
332
                            [% IF export_enabled %]
333
                                <div class="btn-group">
334
                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
335
                                        Export all results<span class="caret"></span>
336
                                    </button>
337
                                    <ul class="dropdown-menu">
338
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
339
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
340
                                        [% FOREACH id IN custom_export_formats.keys %]
341
                                            <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id | uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
342
                                        [% END %]
343
                                   </ul>
344
                                </div> <!-- /.btn-group -->
345
                            [% END %]
346
332
                        </div> <!-- /#selection_ops -->
347
                        </div> <!-- /#selection_ops -->
333
                    </div> <!-- /#searchheader -->
348
                    </div> <!-- /#searchheader -->
334
349
Lines 362-367 Link Here
362
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
377
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
363
                [% END %]
378
                [% END %]
364
379
380
                [% IF export_job_id %]
381
                    <div class="dialog message">
382
                      <p>Exporting records, the export will be processed as soon as possible.</p>
383
                       [% INCLUDE "job_progress.inc" job_id=export_job_id %]
384
                      <p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id | uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
385
                      <div id="job_callback"></div>
386
                    </div>
387
                [% END %]
388
365
                <!-- Search Results Table -->
389
                <!-- Search Results Table -->
366
                [% IF ( total ) %]
390
                [% IF ( total ) %]
367
                    [% IF ( scan ) %]
391
                    [% IF ( scan ) %]
Lines 781-787 Link Here
781
    [% Asset.css("css/humanmsg.css") | $raw %]
805
    [% Asset.css("css/humanmsg.css") | $raw %]
782
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
806
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
783
    [% INCLUDE 'select2.inc' %]
807
    [% INCLUDE 'select2.inc' %]
808
    [% INCLUDE 'str/job_progess.inc' %]
809
    [% Asset.js("js/job_progess.js") | $raw %]
784
    <script>
810
    <script>
811
        [% IF export_job_id %]
812
            updateProgress([% export_job_id | html %], function() {
813
                $.getJSON('/api/v1/jobs/[% export_job_id | html %]', function(job) {
814
                    if (job.data.report.errors.length) {
815
                        humanMsg.displayMsg(
816
                            _("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
817
                            { className: 'humanError' }
818
                        );
819
                    }
820
                    else {
821
                        let export_links = Object.entries(job.data.report.export_links);
822
                        let export_links_html = export_links.map(([format, href]) =>
823
                            `<p>${format}: <a href=${href}>${href}</a></p>`
824
                        ).join('');
825
                        if (export_links.length > 1) {
826
                            export_links_html =
827
                                `<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
828
                        }
829
                        $(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
830
                            .appendTo("#job_callback");
831
                    }
832
                });
833
            });
834
        [% END %]
835
785
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
836
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
786
        var q_array = new Array();  // will hold search terms, if present
837
        var q_array = new Array();  // will hold search terms, if present
787
        var PREF_LocalCoverImages = parseInt( "[% Koha.Preference('LocalCoverImages') | html %]", 10);
838
        var PREF_LocalCoverImages = parseInt( "[% Koha.Preference('LocalCoverImages') | html %]", 10);
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-9 / +109 lines)
Lines 184-193 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
184
184
185
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
185
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
186
186
187
    plan tests => 63;
187
    plan tests => 74;
188
188
189
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
189
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
190
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
190
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
191
191
192
    my @mappings = (
192
    my @mappings = (
193
        {
193
        {
Lines 380-385 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
380
            marc_type => 'marc21',
380
            marc_type => 'marc21',
381
            marc_field => '650(avxyz)',
381
            marc_field => '650(avxyz)',
382
        },
382
        },
383
        {
384
            name => 'local-number',
385
            type => 'string',
386
            facet => 0,
387
            suggestible => 0,
388
            searchable => 1,
389
            sort => 1,
390
            marc_type => 'marc21',
391
            marc_field => '999c',
392
        },
383
    );
393
    );
384
394
385
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
395
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 427-432 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
427
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
437
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
428
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
438
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
429
    );
439
    );
440
430
    my $marc_record_2 = MARC::Record->new();
441
    my $marc_record_2 = MARC::Record->new();
431
    $marc_record_2->leader('     cam  22      a 4500');
442
    $marc_record_2->leader('     cam  22      a 4500');
432
    $marc_record_2->append_fields(
443
    $marc_record_2->append_fields(
Lines 533-539 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
533
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
544
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
534
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
545
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
535
546
536
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
547
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
537
548
538
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
549
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
539
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
550
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 640-647 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
640
        MARC::Field->new('100', '', '', a => 'Author 1'),
651
        MARC::Field->new('100', '', '', a => 'Author 1'),
641
        MARC::Field->new('110', '', '', a => 'Corp Author'),
652
        MARC::Field->new('110', '', '', a => 'Corp Author'),
642
        MARC::Field->new('210', '', '', a => 'Title 1'),
653
        MARC::Field->new('210', '', '', a => 'Title 1'),
643
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
654
        # "|" is for testing escaping for multiple values with custom format
644
        MARC::Field->new('999', '', '', c => '1234567'),
655
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large | record'),
656
        MARC::Field->new('999', '', '', c => '1234569'),
645
    );
657
    );
646
658
647
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
659
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
Lines 654-664 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
654
666
655
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
667
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
656
668
657
    $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
669
    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
658
670
659
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
671
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
660
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
672
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
661
673
674
    # Search export functionality
675
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
676
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
677
    my @es_response_docs;
678
    my $records_data;
679
680
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
681
682
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
683
684
        $docs = $see->marc_records_to_documents(\@source_docs);
685
686
        # Emulate Elasticsearch response docs structure
687
        @es_response_docs = map { { _source => $_ } } @{$docs};
688
689
        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
690
691
        # $large_marc_record should not have been encoded as ISO2709
692
        # since exceeds maximum size, see above
693
        my @tmp = ($marc_record_1, $marc_record_2);
694
        is(
695
            $records_data->{ISO2709},
696
            join('', map { $_->as_usmarc() } @tmp),
697
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
698
        );
699
700
        my $expected_marc_xml = join("\n",
701
            MARC::File::XML::header(),
702
            MARC::File::XML::record($large_marc_record, 'MARC21'),
703
            MARC::File::XML::footer()
704
        );
705
706
        is(
707
            $records_data->{MARCXML},
708
            $expected_marc_xml,
709
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
710
        );
711
712
        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
713
714
        $expected_marc_xml = join("\n",
715
            MARC::File::XML::header(),
716
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
717
            MARC::File::XML::footer()
718
        );
719
720
        is(
721
            $records_data->{MARCXML},
722
            $expected_marc_xml,
723
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
724
        );
725
    }
726
727
    my $custom_formats = <<'END';
728
- name: Biblionumbers
729
  fields: [local-number]
730
  multiple: ignore
731
- name: Title and author
732
  fields: [title, author]
733
  multiple: join
734
END
735
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
736
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
737
738
    # Biblionumbers custom format
739
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
740
    # UTF-8 encode?
741
    is(
742
        $records_data,
743
        "1234567\n1234568\n1234569",
744
        "Records where correctly encoded for the custom format \"Biblionumbers\""
745
    );
746
747
    # Title and author custom format
748
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
749
750
    my $encoded_data = join(
751
        "\n",
752
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
753
        "\"\",\"Author 2\"",
754
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
755
    );
756
757
    is(
758
        $records_data,
759
        $encoded_data,
760
        "Records where correctly encoded for the custom format \"Title and author\""
761
    );
762
662
    push @mappings, {
763
    push @mappings, {
663
        name => 'title',
764
        name => 'title',
664
        type => 'string',
765
        type => 'string',
Lines 791-797 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
791
892
792
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
893
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
793
894
794
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
895
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
795
896
796
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
897
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
797
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
898
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 802-808 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
802
    plan tests => 5;
903
    plan tests => 5;
803
904
804
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
905
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
805
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
906
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
806
907
807
    my $builder = t::lib::TestBuilder->new;
908
    my $builder = t::lib::TestBuilder->new;
808
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
909
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
809
- 

Return to bug 27859