View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/BackgroundJob.pm (+1 lines)
Lines 460-465 sub core_types_to_classes { Link Here
460
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
460
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
461
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
461
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
462
        file_transport_test                 => 'Koha::BackgroundJob::TestTransport',
462
        file_transport_test                 => 'Koha::BackgroundJob::TestTransport',
463
        search_result_export                => 'Koha::BackgroundJob::SearchResultExport',
463
    };
464
    };
464
}
465
}
465
466
(-)a/Koha/BackgroundJob/SearchResultExport.pm (+186 lines)
Line 0 Link Here
1
package Koha::BackgroundJob::SearchResultExport;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
use Try::Tiny;
20
use Koha::SearchEngine::Search;
21
22
use File::Spec;
23
use File::Path qw(mkpath);
24
use Koha::Email;
25
use Koha::UploadedFiles;
26
use POSIX qw(strftime);
27
use Digest::MD5 qw(md5_hex);
28
use Carp qw(croak);
29
use Encode qw(encode);
30
31
use base 'Koha::BackgroundJob';
32
33
=head1 NAME
34
35
Koha::BackgroundJob::SearchResultExport - Export data from search result
36
37
This is a subclass of Koha::BackgroundJob.
38
39
=head1 API
40
41
=head2 Class methods
42
43
=head3 job_type
44
45
Define the job type of this job: stage_marc_for_import
46
47
=cut
48
49
sub job_type {
50
    return 'search_result_export';
51
}
52
53
=head3 process
54
55
Perform the export of search records.
56
57
=cut
58
59
sub process {
60
    my ( $self, $args ) = @_;
61
62
    $self->start;
63
64
    my $data = $self->decoded_data;
65
    my $borrowernumber = $data->{borrowernumber};
66
    my $elasticsearch_query = $args->{elasticsearch_query};
67
    my $preferred_format = $args->{preferred_format};
68
    my $searcher = Koha::SearchEngine::Search->new({
69
        index => $Koha::SearchEngine::BIBLIOS_INDEX
70
    });
71
    my $elasticsearch = $searcher->get_elasticsearch();
72
73
    my $results = eval {
74
        $elasticsearch->search(
75
            index => $searcher->index_name,
76
            scroll => '1m', #TODO: Syspref for scroll time limit?
77
            size => 1000,  #TODO: Syspref for batch size?
78
            body => $elasticsearch_query
79
        );
80
    };
81
    my @errors;
82
    push @errors, $@ if $@;
83
84
    my @docs;
85
    my $encoded_results;
86
    my %export_links;
87
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
88
89
    if (!@errors) {
90
        my $scroll_id = $results->{_scroll_id};
91
        while (@{$results->{hits}->{hits}}) {
92
            push @docs, @{$results->{hits}->{hits}};
93
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
94
            $results = $elasticsearch->scroll(
95
                scroll => '1m',
96
                scroll_id => $scroll_id
97
            );
98
        }
99
100
        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
101
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
102
        }
103
        else {
104
            $encoded_results->{$preferred_format->{name}} =
105
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
106
        }
107
108
        my %format_extensions = (
109
            'ISO2709' => '.mrc',
110
            'MARCXML' => '.xml',
111
        );
112
113
        my $upload_dir = Koha::UploadedFile->permanent_directory;
114
115
        while (my ($format, $data) = each %{$encoded_results}) {
116
            my $hash = md5_hex($data);
117
            my $category = "search_marc_export";
118
            my $time = strftime "%Y%m%d_%H%M", localtime time;
119
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
120
            my $filename = $category . '_' . $time . $ext;
121
            my $file_dir = File::Spec->catfile($upload_dir, $category);
122
            if ( !-d $file_dir) {
123
                unless(mkpath $file_dir) {
124
                    push @errors, "Failed to create $file_dir";
125
                    next;
126
                }
127
            }
128
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
129
130
            my $fh = IO::File->new($filepath, "w");
131
132
            if ($fh) {
133
                $fh->binmode;
134
                print $fh encode('UTF-8', $data);
135
                $fh->close;
136
137
                my $size = -s $filepath;
138
                my $file = Koha::UploadedFile->new({
139
                        hashvalue => $hash,
140
                        filename  => $filename,
141
                        dir       => $category,
142
                        filesize  => $size,
143
                        owner     => $borrowernumber,
144
                        uploadcategorycode => 'search_marc_export',
145
                        public    => 0,
146
                        permanent => 1,
147
                    })->store;
148
                my $id = $file->_result()->get_column('id');
149
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
150
            }
151
            else {
152
                push @errors, "Failed to write \"$filepath\"";
153
            }
154
        }
155
    }
156
    my $report = {
157
        export_links => \%export_links,
158
        total => scalar @docs,
159
        errors => \@errors,
160
        query_string => $query_string,
161
    };
162
    $data->{report}   = $report;
163
    if (@errors) {
164
        $self->set({ progress => 0, status => 'failed' })->store;
165
    }
166
    else {
167
        $self->finish($data);
168
    }
169
}
170
171
=head3 enqueue
172
173
Enqueue the new job
174
175
=cut
176
177
sub enqueue {
178
    my ( $self, $args) = @_;
179
    $self->SUPER::enqueue({
180
        job_size => $args->{size},
181
        job_args => $args,
182
        job_queue => 'long_tasks',
183
    });
184
}
185
186
1;
(-)a/Koha/SearchEngine/Elasticsearch.pm (-48 / +317 lines)
Lines 43-50 use YAML::XS; Link Here
43
43
44
use List::Util qw( sum0 );
44
use List::Util qw( sum0 );
45
use MARC::File::XML;
45
use MARC::File::XML;
46
use MIME::Base64 qw( encode_base64 );
46
use MIME::Base64 qw(encode_base64 decode_base64);
47
use Encode       qw( encode );
47
use Encode qw(encode decode);
48
48
use Business::ISBN;
49
use Business::ISBN;
49
use Scalar::Util qw( looks_like_number );
50
use Scalar::Util qw( looks_like_number );
50
51
Lines 631-637 sub marc_records_to_documents { Link Here
631
    my $control_fields_rules = $rules->{control_fields};
632
    my $control_fields_rules = $rules->{control_fields};
632
    my $data_fields_rules    = $rules->{data_fields};
633
    my $data_fields_rules    = $rules->{data_fields};
633
    my $marcflavour          = lc C4::Context->preference('marcflavour');
634
    my $marcflavour          = lc C4::Context->preference('marcflavour');
634
    my $use_array            = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
635
635
636
    my @record_documents;
636
    my @record_documents;
637
637
Lines 894-941 sub marc_records_to_documents { Link Here
894
            }
894
            }
895
        }
895
        }
896
896
897
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
897
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
898
        $record->encoding('UTF-8');
898
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
899
        if ($use_array) {
899
            $record,
900
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
900
            $preferred_format,
901
            $record_document->{'marc_format'}     = 'ARRAY';
901
            $marcflavour
902
        } else {
902
        );
903
            my @warnings;
904
            {
905
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
906
                local $SIG{__WARN__} = sub {
907
                    push @warnings, $_[0];
908
                };
909
                my $usmarc_record = $record->as_usmarc();
910
911
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
912
                my $decoded_usmarc_record;
913
                eval { $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); };
914
                if ( $@ || $decoded_usmarc_record->warnings() ) {
915
916
                    #NOTE: We override the warnings since they're many and misleading
917
                    @warnings = (
918
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
919
                    );
920
                }
921
922
                my $marc_data = encode_base64( encode( 'UTF-8', $usmarc_record ) );
923
                $record_document->{'marc_data'} = $marc_data;
924
            }
925
            if (@warnings) {
926
903
927
                # Suppress warnings if record length exceeded
904
        if ($preferred_format eq 'ARRAY') {
928
                unless ( substr( $record->leader(), 0, 5 ) eq '99999' ) {
905
            $record_document->{'marc_data_array'} = $encoded_record;
929
                    foreach my $warning (@warnings) {
906
        } else {
930
                        carp $warning;
907
            $record_document->{'marc_data'} = $encoded_record;
931
                    }
932
                }
933
                $record_document->{'marc_data'}   = $record->as_xml_record($marcflavour);
934
                $record_document->{'marc_format'} = 'MARCXML';
935
            } else {
936
                $record_document->{'marc_format'} = 'base64ISO2709';
937
            }
938
        }
908
        }
909
        $record_document->{'marc_format'} = $format;
939
910
940
        if (   $self->index eq $AUTHORITIES_INDEX
911
        if (   $self->index eq $AUTHORITIES_INDEX
941
            && exists $record_document->{'subject-heading-thesaurus'}
912
            && exists $record_document->{'subject-heading-thesaurus'}
Lines 979-984 sub marc_records_to_documents { Link Here
979
    return \@record_documents;
950
    return \@record_documents;
980
}
951
}
981
952
953
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
954
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
955
956
Encode a MARC::Record to the preferred marc document record format. If record
957
exceeds ISO2709 maximum size record size and C<$format> is set to
958
'base64ISO2709' format will fallback to 'MARCXML' instead.
959
960
=over 4
961
962
=item C<$record>
963
964
A MARC::Record object
965
966
=item C<$marcflavour>
967
968
The marcflavour to use
969
970
=back
971
972
=cut
973
974
sub search_document_marc_record_encode {
975
    my ($self, $record, $format, $marcflavour) = @_;
976
977
    $record->encoding('UTF-8');
978
979
    if ($format eq 'ARRAY') {
980
        return ($self->_marc_to_array($record), $format);
981
    }
982
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
983
        my @warnings;
984
        my $marc_data;
985
        # Save origial leader since as_usmarc will modify leader
986
        # resulting in failed tests when comparing records
987
        my $original_leader = $record->leader();
988
        {
989
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
990
            local $SIG{__WARN__} = sub {
991
                push @warnings, $_[0];
992
            };
993
            $marc_data = $record->as_usmarc();
994
            if (!@warnings) {
995
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
996
                my $usmarc_record;
997
                eval { $usmarc_record = MARC::Record->new_from_usmarc($marc_data); };
998
                @warnings = $usmarc_record->warnings() if defined $usmarc_record;
999
                if ($@ || @warnings) {
1000
                    #NOTE: We override the warnings since they're many and misleading
1001
                    @warnings = (
1002
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
1003
                    );
1004
                }
1005
            }
1006
        }
1007
        if (@warnings) {
1008
            # Suppress warnings if record length exceeded
1009
            unless (substr($record->leader(), 0, 5) eq '99999') {
1010
                foreach my $warning (@warnings) {
1011
                    carp $warning;
1012
                }
1013
            }
1014
            # Restore original leader
1015
            $record->leader($original_leader);
1016
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
1017
        }
1018
        else {
1019
            $marc_data = encode('UTF-8', $marc_data);
1020
            if ($format eq 'base64ISO2709') {
1021
                $marc_data = encode_base64($marc_data);
1022
            }
1023
            return ($marc_data, $format);
1024
        }
1025
    }
1026
    elsif ($format eq 'MARCXML') {
1027
        return (MARC::File::XML::record($record, $marcflavour), $format);
1028
    }
1029
    else {
1030
        # This should be unlikely to happen
1031
        croak "Invalid marc record serialization format: $format";
1032
    }
1033
}
1034
1035
=head2 search_document_marc_record_decode
1036
    my $marc_record = $self->search_document_marc_record_decode(@result);
1037
1038
Extract marc data from Elasticsearch result and decode to MARC::Record object
1039
1040
=cut
1041
1042
sub search_document_marc_record_decode {
1043
    # Result is passed in as array, will get flattened
1044
    # and first element will be $result
1045
    my ($self, $result) = @_;
1046
    if ($result->{marc_format} eq 'base64ISO2709') {
1047
        my $marc_data = decode('utf-8', decode_base64($result->{marc_data}));
1048
        my $record = MARC::Record->new_from_usmarc($marc_data);
1049
        return $record;
1050
    }
1051
    elsif ($result->{marc_format} eq 'MARCXML') {
1052
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
1053
    }
1054
    elsif ($result->{marc_format} eq 'ARRAY') {
1055
        return $self->_array_to_marc($result->{marc_data_array});
1056
    }
1057
    else {
1058
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
1059
    }
1060
}
1061
1062
=head2 search_documents_encode($docs, $preferred_format)
1063
1064
    $records_data = $self->search_documents_encode($docs, $preferred_format)
1065
1066
Return marc encoded records from ElasticSearch search result documents. The return value
1067
C<$marc_records> is a hashref with encoded records keyed by MARC format.
1068
1069
=over 4
1070
1071
=item C<$docs>
1072
1073
An arrayref of Elasticsearch search documents
1074
1075
=item C<$preferred_format>
1076
1077
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
1078
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
1079
is set to 'ISO2709'.
1080
1081
=back
1082
1083
=cut
1084
1085
sub search_documents_encode {
1086
1087
    my ($self, $docs, $preferred_format) = @_;
1088
1089
    my %encoded_records = (
1090
        'ISO2709' => [],
1091
        'MARCXML' => []
1092
    );
1093
1094
    unless (exists $encoded_records{$preferred_format}) {
1095
       croak "Invalid preferred format: $preferred_format";
1096
    }
1097
1098
    for my $es_record (@{$docs}) {
1099
        # Special optimized cases
1100
        my $marc_data;
1101
        my $resulting_format = $preferred_format;
1102
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
1103
            $marc_data = $es_record->{_source}{marc_data};
1104
        }
1105
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
1106
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
1107
        }
1108
        else {
1109
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
1110
            my $marcflavour = lc C4::Context->preference('marcflavour');
1111
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
1112
        }
1113
        push @{$encoded_records{$resulting_format}}, $marc_data;
1114
    }
1115
    if (@{$encoded_records{'ISO2709'}}) {
1116
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
1117
    }
1118
    else {
1119
        delete $encoded_records{'ISO2709'};
1120
    }
1121
1122
    if (@{$encoded_records{'MARCXML'}}) {
1123
        $encoded_records{'MARCXML'} = join(
1124
            "\n",
1125
            MARC::File::XML::header(),
1126
            join("\n", @{$encoded_records{'MARCXML'}}),
1127
            MARC::File::XML::footer()
1128
        );
1129
    }
1130
    else {
1131
        delete $encoded_records{'MARCXML'};
1132
    }
1133
1134
    return \%encoded_records;
1135
}
1136
1137
=head2 search_result_export_custom_formats()
1138
1139
    $custom_formats = $self->search_result_export_custom_formats()
1140
1141
Return user defined custom search result export formats.
1142
1143
=cut
1144
1145
sub search_result_export_custom_formats {
1146
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
1147
    my $custom_export_formats = {};
1148
1149
    if (ref $export_custom_formats_pref eq 'ARRAY') {
1150
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
1151
            # TODO: Perhaps validate on save or trow error here instead of just
1152
            # ignoring invalid formats
1153
            my $format = $export_custom_formats_pref->[$i];
1154
            if (
1155
                ref $format->{fields} eq 'ARRAY' &&
1156
                @{$format->{fields}} &&
1157
                $format->{name}
1158
            ) {
1159
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
1160
                $custom_export_formats->{"custom_$i"} = $format;
1161
            }
1162
        }
1163
    }
1164
    return $custom_export_formats;
1165
}
1166
1167
=head2 search_documents_custom_format_encode($docs, $custom_format)
1168
1169
    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
1170
1171
Return encoded records from ElasticSearch search result documents using a
1172
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
1173
Returns the encoded records.
1174
1175
=over 4
1176
1177
=item C<$docs>
1178
1179
An arrayref of Elasticsearch search documents
1180
1181
=item C<$format>
1182
1183
A hashref with the custom format definition.
1184
1185
=back
1186
1187
=cut
1188
1189
sub search_documents_custom_format_encode {
1190
    my ($self, $docs, $format) = @_;
1191
1192
    my $result;
1193
1194
    my $doc_get_fields = sub {
1195
        my ($doc, $fields) = @_;
1196
        my @row;
1197
        foreach my $field (@{$fields}) {
1198
            my $values = $doc->{_source}->{$field};
1199
            push @row, ref $values eq 'ARRAY' ? $values : [''];
1200
        }
1201
        return \@row;
1202
    };
1203
1204
    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1205
1206
    if($format->{multiple} eq 'ignore') {
1207
        for (my $i = 0; $i < @rows; ++$i) {
1208
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1209
        }
1210
    }
1211
    elsif($format->{multiple} eq 'newline') {
1212
        if (@{$format->{fields}} == 1) {
1213
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
1214
        }
1215
        else {
1216
            croak "'newline' is only valid for single field export formats";
1217
        }
1218
    }
1219
    elsif($format->{multiple} eq 'join') {
1220
        for (my $i = 0; $i < @rows; ++$i) {
1221
            # Escape separator
1222
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1223
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1224
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
1225
                }
1226
            }
1227
            # Separate multiple values with "|"
1228
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
1229
        }
1230
    }
1231
    else {
1232
        croak "Invalid 'multiple' option: " . $format->{multiple};
1233
    }
1234
    if (@{$format->{fields}} == 1) {
1235
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
1236
    }
1237
    else {
1238
        # Encode CSV
1239
        for (my $i = 0; $i < @rows; ++$i) {
1240
            # Escape quotes
1241
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1242
                $rows[$i][$j] =~ s/"/""/g;
1243
            }
1244
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1245
        }
1246
    }
1247
1248
    return join("\n", @rows);
1249
}
1250
982
=head2 _marc_to_array($record)
1251
=head2 _marc_to_array($record)
983
1252
984
    my @fields = _marc_to_array($record)
1253
    my @fields = _marc_to_array($record)
Lines 1050-1067 sub _array_to_marc { Link Here
1050
    $record->leader( $data->{leader} );
1319
    $record->leader( $data->{leader} );
1051
    for my $field ( @{ $data->{fields} } ) {
1320
    for my $field ( @{ $data->{fields} } ) {
1052
        my $tag = ( keys %{$field} )[0];
1321
        my $tag = ( keys %{$field} )[0];
1053
        $field = $field->{$tag};
1322
        my $field_data = $field->{$tag};
1054
        my $marc_field;
1323
        my $marc_field;
1055
        if ( ref($field) eq 'HASH' ) {
1324
        if ( ref($field_data) eq 'HASH' ) {
1056
            my @subfields;
1325
            my @subfields;
1057
            foreach my $subfield ( @{ $field->{subfields} } ) {
1326
            foreach my $subfield ( @{ $field_data->{subfields} } ) {
1058
                my $code = ( keys %{$subfield} )[0];
1327
                my $code = ( keys %{$subfield} )[0];
1059
                push @subfields, $code;
1328
                push @subfields, $code;
1060
                push @subfields, $subfield->{$code};
1329
                push @subfields, $subfield->{$code};
1061
            }
1330
            }
1062
            $marc_field = MARC::Field->new( $tag, $field->{ind1}, $field->{ind2}, @subfields );
1331
            $marc_field = MARC::Field->new( $tag, $field_data->{ind1}, $field_data->{ind2}, @subfields );
1063
        } else {
1332
        } else {
1064
            $marc_field = MARC::Field->new( $tag, $field );
1333
            $marc_field = MARC::Field->new( $tag, $field_data );
1065
        }
1334
        }
1066
        $record->append_fields($marc_field);
1335
        $record->append_fields($marc_field);
1067
    }
1336
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-26 / +3 lines)
Lines 180-186 sub search_compat { Link Here
180
    my $index = $offset;
180
    my $index = $offset;
181
    my $hits  = $results->{'hits'};
181
    my $hits  = $results->{'hits'};
182
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
182
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
183
        $records[ $index++ ] = $self->decode_record_from_result( $es_record->{'_source'} );
183
        $records[ $index++ ] = $self->search_document_marc_record_decode( $es_record->{'_source'} );
184
    }
184
    }
185
185
186
    # consumers of this expect a name-spaced result, we provide the default
186
    # consumers of this expect a name-spaced result, we provide the default
Lines 248-254 sub search_auth_compat { Link Here
248
            # it's not reproduced here yet.
248
            # it's not reproduced here yet.
249
            my $authtype           = $rs->single;
249
            my $authtype           = $rs->single;
250
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
250
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
251
            my $marc               = $self->decode_record_from_result($record);
251
            my $marc               = $self->search_document_marc_record_decode($record);
252
            my $mainentry          = $marc->field($auth_tag_to_report);
252
            my $mainentry          = $marc->field($auth_tag_to_report);
253
            my $reported_tag;
253
            my $reported_tag;
254
            if ($mainentry) {
254
            if ($mainentry) {
Lines 387-393 sub simple_search_compat { Link Here
387
    my @records;
387
    my @records;
388
    my $hits = $results->{'hits'};
388
    my $hits = $results->{'hits'};
389
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
389
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
390
        push @records, $self->decode_record_from_result( $es_record->{'_source'} );
390
        push @records, $self->search_document_marc_record_decode( $es_record->{'_source'} );
391
    }
391
    }
392
    return ( undef, \@records, $hits->{'total'} );
392
    return ( undef, \@records, $hits->{'total'} );
393
}
393
}
Lines 407-435 sub extract_biblionumber { Link Here
407
    return Koha::SearchEngine::Search::extract_biblionumber($searchresultrecord);
407
    return Koha::SearchEngine::Search::extract_biblionumber($searchresultrecord);
408
}
408
}
409
409
410
=head2 decode_record_from_result
411
    my $marc_record = $self->decode_record_from_result(@result);
412
413
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
414
415
=cut
416
417
sub decode_record_from_result {
418
419
    # Result is passed in as array, will get flattened
420
    # and first element will be $result
421
    my ( $self, $result ) = @_;
422
    if ( $result->{marc_format} eq 'base64ISO2709' ) {
423
        return MARC::Record->new_from_usmarc( decode_base64( $result->{marc_data} ) );
424
    } elsif ( $result->{marc_format} eq 'MARCXML' ) {
425
        return MARC::Record->new_from_xml( $result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour') );
426
    } elsif ( $result->{marc_format} eq 'ARRAY' ) {
427
        return $self->_array_to_marc( $result->{marc_data_array} );
428
    } else {
429
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
430
    }
431
}
432
433
=head2 max_result_window
410
=head2 max_result_window
434
411
435
Returns the maximum number of results that can be fetched
412
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+45 lines)
Lines 148-153 use C4::Languages qw( getlanguage getLanguages ); Link Here
148
use C4::Koha        qw( getitemtypeimagelocation GetAuthorisedValues );
148
use C4::Koha        qw( getitemtypeimagelocation GetAuthorisedValues );
149
use URI::Escape;
149
use URI::Escape;
150
use POSIX qw(ceil floor);
150
use POSIX qw(ceil floor);
151
use Carp qw(croak);
151
152
152
use Koha::ItemTypes;
153
use Koha::ItemTypes;
153
use Koha::Library::Groups;
154
use Koha::Library::Groups;
Lines 157-162 use Koha::SearchEngine::QueryBuilder; Link Here
157
use Koha::Virtualshelves;
158
use Koha::Virtualshelves;
158
use Koha::SearchFields;
159
use Koha::SearchFields;
159
use Koha::SearchFilters;
160
use Koha::SearchFilters;
161
use Koha::BackgroundJob::SearchResultExport;
160
162
161
use URI::Escape;
163
use URI::Escape;
162
use JSON qw( decode_json encode_json );
164
use JSON qw( decode_json encode_json );
Lines 760-765 $template->param( Link Here
760
    add_to_some_public_shelves  => $some_public_shelves,
762
    add_to_some_public_shelves  => $some_public_shelves,
761
);
763
);
762
764
765
my $patron = Koha::Patrons->find( $borrowernumber );
766
my $export_enabled =
767
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
768
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
769
    $patron && $patron->has_permission({ tools => 'export_catalog' });
770
771
$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
772
773
if ($export_enabled) {
774
775
    my $export = $cgi->param('export');
776
    my $preferred_format = $cgi->param('export_format');
777
    my $custom_export_formats = $searcher->search_result_export_custom_formats;
778
779
    $template->param(custom_export_formats => $custom_export_formats);
780
781
    # TODO: Need to handle $hits = 0?
782
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
783
784
    if ($export && $preferred_format && $hits) {
785
        unless (
786
            $preferred_format eq 'ISO2709' ||
787
            $preferred_format eq 'MARCXML'
788
        ) {
789
            if (!exists $custom_export_formats->{$preferred_format}) {
790
                croak "Invalid export format: $preferred_format";
791
            }
792
            else {
793
                $preferred_format = $custom_export_formats->{$preferred_format};
794
            }
795
        }
796
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
797
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
798
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
799
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
800
            size => $size,
801
            preferred_format => $preferred_format,
802
            elasticsearch_query => \%export_query
803
        });
804
        $template->param(export_job_id => $export_job_id);
805
    }
806
}
807
763
output_html_with_http_headers $cgi, $cookie, $template->output;
808
output_html_with_http_headers $cgi, $cookie, $template->output;
764
809
765
=head2 prepare_adv_search_types
810
=head2 prepare_adv_search_types
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+19 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', '1', NULL, 'Enable search result export', 'YesNo') });
11
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
12
13
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
14
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
15
16
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', '0', NULL, 'Search result export limit', 'integer') });
17
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
18
    },
19
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (+3 lines)
Lines 257-262 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
257
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
257
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
258
('EnableBooking','1',NULL,'If enabled, activate every functionnalities related with Bookings module','YesNo'),
258
('EnableBooking','1',NULL,'If enabled, activate every functionnalities related with Bookings module','YesNo'),
259
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
259
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
260
('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
260
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
261
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
261
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
262
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
262
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
263
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
Lines 724-729 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
724
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
725
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
725
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
726
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
726
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
727
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
728
('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
729
('ElasticsearchSearchResultExportLimit', '0', NULL, 'Search result export limit', 'integer'),
727
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
730
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
728
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
731
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
729
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
732
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
Line 0 Link Here
1
[% USE Koha %]
2
3
[% BLOCK report %]
4
    [% SET report = job.report %]
5
    [% IF report %]
6
        [% IF job.status == 'finished' %]
7
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
8
            <ul>
9
              <li>[% report.total | html %] records exported</li>
10
              [% IF report.export_links.keys.size > 1 %]
11
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
12
              [% END %]
13
              [% FOREACH format IN report.export_links.keys.sort %]
14
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
15
              [% END %]
16
            </ul>
17
        [% ELSE %]
18
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
19
            <ul>
20
            [% FOREACH error in report.errors %]
21
              <li>[% error | html %]</li>
22
            [% END %]
23
            </ul>
24
        [% END %]
25
    [% END %]
26
[% END %]
27
28
[% BLOCK detail %]
29
[% END %]
30
31
[% BLOCK js %]
32
[% END %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
Lines 230-235 Link Here
230
                '_id': 'file_transport_test',
230
                '_id': 'file_transport_test',
231
                '_str': _("File transport connection test")
231
                '_str': _("File transport connection test")
232
            },
232
            },
233
            {
234
                '_id': 'search_result_export',
235
                '_str': _("Search result export")
236
            },
233
            [% FOR job_type IN plugin_job_types %]
237
            [% FOR job_type IN plugin_job_types %]
234
            {
238
            {
235
                '_id': '[% job_type.id | html %]',
239
                '_id': '[% job_type.id | html %]',
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
Lines 175-180 Searching: Link Here
175
                  1: use
175
                  1: use
176
                  0: "don't use"
176
                  0: "don't use"
177
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
177
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
178
        -
179
            - pref: EnableElasticsearchSearchResultExport
180
              type: boolean
181
              default: yes
182
              choices:
183
                  1: Enable
184
                  0: Disable
185
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
186
        -
187
            - pref: ElasticsearchSearchResultExportCustomFormats
188
              type: textarea
189
              syntax: text/x-yaml
190
              class: code
191
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
192
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
193
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
194
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
195
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
196
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
197
            - <p>The behavior when handling fields with multiple values.</p>
198
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
199
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
200
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
201
            - 'Example:</br>'
202
            - '- name: Biblionumbers<br />'
203
            - '&nbsp;&nbsp;fields: [local-number]<br />'
204
            - '&nbsp;&nbsp;multiple: ignore<br />'
205
            - '- name: Title and author<br />'
206
            - '&nbsp;&nbsp;fields: [title, author]<br />'
207
            - '&nbsp;&nbsp;multiple: join<br /><br />'
208
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
209
        -
210
            - Limit export from search results to a maximum of
211
            - pref: ElasticsearchSearchResultExportLimit
212
              class: integer
213
            - search result items (Elasticsearch only).<br /><br />
214
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
178
    Results display:
215
    Results display:
179
        -
216
        -
180
            - pref: numSearchResultsDropdown
217
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+52 lines)
Lines 363-368 Link Here
363
                    </div>
363
                    </div>
364
                    <!-- /.btn-group -->
364
                    <!-- /.btn-group -->
365
365
366
                    [% IF export_enabled %]
367
                        <div class="btn-group">
368
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
369
                                Export all results<span class="caret"></span>
370
                            </button>
371
                            <ul class="dropdown-menu">
372
                                <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
373
                                <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
374
                                [% FOREACH id IN custom_export_formats.keys %]
375
                                    <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id | uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
376
                                [% END %]
377
                           </ul>
378
                        </div> <!-- /.btn-group -->
379
                    [% END %]
380
366
                    [% IF Koha.Preference('numSearchResultsDropdown') %]
381
                    [% IF Koha.Preference('numSearchResultsDropdown') %]
367
                        <div class="btn-group">
382
                        <div class="btn-group">
368
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Results per page: [% results_per_page | html %] </button>
383
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Results per page: [% results_per_page | html %] </button>
Lines 452-457 Link Here
452
        >
467
        >
453
    [% END %]
468
    [% END %]
454
469
470
    [% IF export_job_id %]
471
        <div class="dialog message">
472
          <p>Exporting records, the export will be processed as soon as possible.</p>
473
           [% INCLUDE "job_progress.inc" job_id=export_job_id %]
474
          <p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id | uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
475
          <div id="job_callback"></div>
476
        </div>
477
    [% END %]
478
479
455
    <!-- Search Results Table -->
480
    <!-- Search Results Table -->
456
    [% IF ( total ) %]
481
    [% IF ( total ) %]
457
        [% IF ( scan ) %]
482
        [% IF ( scan ) %]
Lines 965-971 Link Here
965
    [% Asset.css("css/humanmsg.css") | $raw %]
990
    [% Asset.css("css/humanmsg.css") | $raw %]
966
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
991
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
967
    [% INCLUDE 'select2.inc' %]
992
    [% INCLUDE 'select2.inc' %]
993
    [% INCLUDE 'str/job_progress.inc' %]
994
    [% Asset.js("js/job_progress.js") | $raw %]
968
    <script>
995
    <script>
996
        [% IF export_job_id %]
997
            updateProgress([% export_job_id | html %], function() {
998
                $.getJSON('/api/v1/jobs/[% export_job_id | html %]', function(job) {
999
                    if (job.data.report.errors.length) {
1000
                        humanMsg.displayMsg(
1001
                            _("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
1002
                            { className: 'humanError' }
1003
                        );
1004
                    }
1005
                    else {
1006
                        let export_links = Object.entries(job.data.report.export_links);
1007
                        let export_links_html = export_links.map(([format, href]) =>
1008
                            `<p>${format}: <a href=${href}>${href}</a></p>`
1009
                        ).join('');
1010
                        if (export_links.length > 1) {
1011
                            export_links_html =
1012
                                `<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
1013
                        }
1014
                        $(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
1015
                            .appendTo("#job_callback");
1016
                    }
1017
                });
1018
            });
1019
        [% END %]
1020
969
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
1021
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
970
        var q_array = new Array();  // will hold search terms, if present
1022
        var q_array = new Array();  // will hold search terms, if present
971
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
1023
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-20 / +123 lines)
Lines 16-22 Link Here
16
# along with Koha; if not, see <https://www.gnu.org/licenses>.
16
# along with Koha; if not, see <https://www.gnu.org/licenses>.
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
use Encode;
19
use utf8;
20
20
21
use Test::More tests => 9;
21
use Test::More tests => 9;
22
use Test::NoWarnings;
22
use Test::NoWarnings;
Lines 211-217 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
211
211
212
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
212
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
213
213
214
    plan tests => 71;
214
    plan tests => 82;
215
215
216
    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
216
    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
217
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );
217
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );
Lines 428-433 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
428
            marc_type   => 'marc21',
428
            marc_type   => 'marc21',
429
            marc_field  => '522a',
429
            marc_field  => '522a',
430
        },
430
        },
431
        {
432
            name => 'local-number',
433
            type => 'string',
434
            facet => 0,
435
            suggestible => 0,
436
            searchable => 1,
437
            sort => 1,
438
            marc_type => 'marc21',
439
            marc_field => '999c',
440
        },
431
    );
441
    );
432
442
433
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
443
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 460-466 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
460
    my $long_callno = '1234567890' x 30;
470
    my $long_callno = '1234567890' x 30;
461
471
462
    my $marc_record_1 = MARC::Record->new();
472
    my $marc_record_1 = MARC::Record->new();
463
    $marc_record_1->leader('     cam  22      a 4500');
473
    $marc_record_1->leader('     cam a22      a 4500');
464
    $marc_record_1->append_fields(
474
    $marc_record_1->append_fields(
465
        MARC::Field->new( '001', '123' ),
475
        MARC::Field->new( '001', '123' ),
466
        MARC::Field->new( '007', 'ku' ),
476
        MARC::Field->new( '007', 'ku' ),
Lines 481-488 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
481
        MARC::Field->new( '952', '', '', 0 => 0,    g => '127.20', o => $callno2,     l => 2 ),
491
        MARC::Field->new( '952', '', '', 0 => 0,    g => '127.20', o => $callno2,     l => 2 ),
482
        MARC::Field->new( '952', '', '', 0 => 1,    g => '0.00',   o => $long_callno, l => 1 ),
492
        MARC::Field->new( '952', '', '', 0 => 1,    g => '0.00',   o => $long_callno, l => 1 ),
483
    );
493
    );
494
484
    my $marc_record_2 = MARC::Record->new();
495
    my $marc_record_2 = MARC::Record->new();
485
    $marc_record_2->leader('     cam  22      a 4500');
496
    $marc_record_2->leader('     cam a22      a 4500');
486
    $marc_record_2->append_fields(
497
    $marc_record_2->append_fields(
487
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
498
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
488
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
499
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
Lines 495-501 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
495
    );
506
    );
496
507
497
    my $marc_record_3 = MARC::Record->new();
508
    my $marc_record_3 = MARC::Record->new();
498
    $marc_record_3->leader('     cam  22      a 4500');
509
    $marc_record_3->leader('     cam a22      a 4500');
499
    $marc_record_3->append_fields(
510
    $marc_record_3->append_fields(
500
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
511
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
501
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
512
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
Lines 508-514 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
508
    );
519
    );
509
520
510
    my $marc_record_4 = MARC::Record->new();
521
    my $marc_record_4 = MARC::Record->new();
511
    $marc_record_4->leader('     cam  22      a 4500');
522
    $marc_record_4->leader('     cam a22      a 4500');
512
    $marc_record_4->append_fields(
523
    $marc_record_4->append_fields(
513
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
524
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
514
        MARC::Field->new( '100', '', '',  a => 'Author 2' ),
525
        MARC::Field->new( '100', '', '',  a => 'Author 2' ),
Lines 630-636 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
630
    ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' );
641
    ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' );
631
    is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' );
642
    is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' );
632
643
633
    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
644
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
634
645
635
    ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" );
646
    ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" );
636
    is(
647
    is(
Lines 749-768 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
749
    # Marc serialization format fallback for records exceeding base64ISO2709 max record size
760
    # Marc serialization format fallback for records exceeding base64ISO2709 max record size
750
761
751
    my $large_marc_record = MARC::Record->new();
762
    my $large_marc_record = MARC::Record->new();
752
    $large_marc_record->leader('     cam  22      a 4500');
763
    $large_marc_record->leader('     cam a22      a 4500');
753
764
754
    $large_marc_record->append_fields(
765
    $large_marc_record->append_fields(
755
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
766
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
756
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
767
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
757
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
768
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
758
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large record' ),
769
        # "|" is for testing escaping for multiple values with custom format
759
        MARC::Field->new( '999', '', '', c => '1234567' ),
770
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large | record' ),
771
        MARC::Field->new( '999', '', '', c => '1234569' ),
760
    );
772
    );
761
773
762
    my $item_field = MARC::Field->new(
774
    my $item_field = MARC::Field->new(
763
        '952', '', '', o => '123456789123456789123456789', p => '123456789',
775
        '952', '', '', o => '123456789123456789123456789', p => '123456789',
764
        z => Encode::decode( 'UTF-8', 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.' )
776
        z => 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.'
765
    );
777
    );
778
766
    my $items_count = 1638;
779
    my $items_count = 1638;
767
    while ( --$items_count ) {
780
    while ( --$items_count ) {
768
        $large_marc_record->append_fields($item_field);
781
        $large_marc_record->append_fields($item_field);
Lines 775-781 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
775
        'For record exceeding max record size marc_format should be set correctly'
788
        'For record exceeding max record size marc_format should be set correctly'
776
    );
789
    );
777
790
778
    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
791
    $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
779
792
780
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
793
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
781
    is(
794
    is(
Lines 783-788 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
783
        "Decoded MARCXML record has same data as original record"
796
        "Decoded MARCXML record has same data as original record"
784
    );
797
    );
785
798
799
    # Search export functionality
800
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
801
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
802
    my @es_response_docs;
803
    my $records_data;
804
805
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
806
807
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
808
809
        $docs = $see->marc_records_to_documents(\@source_docs);
810
811
        # Emulate Elasticsearch response docs structure
812
        @es_response_docs = map { { _source => $_ } } @{$docs};
813
814
        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
815
816
        # $large_marc_record should not have been encoded as ISO2709
817
        # since exceeds maximum size, see above
818
        my @tmp = ($marc_record_1, $marc_record_2);
819
        is(
820
            $records_data->{ISO2709},
821
            join('', map { $_->as_usmarc() } @tmp),
822
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
823
        );
824
825
        my $expected_marc_xml = join("\n",
826
            MARC::File::XML::header(),
827
            MARC::File::XML::record($large_marc_record, 'MARC21'),
828
            MARC::File::XML::footer()
829
        );
830
831
        is(
832
            $records_data->{MARCXML},
833
            $expected_marc_xml,
834
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
835
        );
836
837
        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
838
839
        $expected_marc_xml = join("\n",
840
            MARC::File::XML::header(),
841
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
842
            MARC::File::XML::footer()
843
        );
844
845
        is(
846
            $records_data->{MARCXML},
847
            $expected_marc_xml,
848
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
849
        );
850
    }
851
852
    my $custom_formats = <<'END';
853
- name: Biblionumbers
854
  fields: [local-number]
855
  multiple: ignore
856
- name: Title and author
857
  fields: [title, author]
858
  multiple: join
859
END
860
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
861
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
862
863
    # Biblionumbers custom format
864
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
865
    # UTF-8 encode?
866
    is(
867
        $records_data,
868
        "1234567\n1234568\n1234569",
869
        "Records where correctly encoded for the custom format \"Biblionumbers\""
870
    );
871
872
    # Title and author custom format
873
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
874
875
    my $encoded_data = join(
876
        "\n",
877
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
878
        "\"\",\"Author 2\"",
879
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
880
    );
881
882
    is(
883
        $records_data,
884
        $encoded_data,
885
        "Records where correctly encoded for the custom format \"Title and author\""
886
    );
887
786
    push @mappings, {
888
    push @mappings, {
787
        name        => 'title',
889
        name        => 'title',
788
        type        => 'string',
890
        type        => 'string',
Lines 826-832 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
826
928
827
    pop @mappings;
929
    pop @mappings;
828
    my $marc_record_with_blank_field = MARC::Record->new();
930
    my $marc_record_with_blank_field = MARC::Record->new();
829
    $marc_record_with_blank_field->leader('     cam  22      a 4500');
931
    $marc_record_with_blank_field->leader('     cam a22      a 4500');
830
932
831
    $marc_record_with_blank_field->append_fields(
933
    $marc_record_with_blank_field->append_fields(
832
        MARC::Field->new( '100', '', '', a => '' ),
934
        MARC::Field->new( '100', '', '', a => '' ),
Lines 839-845 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
839
    is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' );
941
    is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' );
840
942
841
    my $marc_record_with_large_field = MARC::Record->new();
943
    my $marc_record_with_large_field = MARC::Record->new();
842
    $marc_record_with_large_field->leader('     cam  22      a 4500');
944
    $marc_record_with_large_field->leader('     cam a22      a 4500');
843
945
844
    my $xs = 'X' x 8191;
946
    my $xs = 'X' x 8191;
845
    my $ys = 'Y' x 8191;
947
    my $ys = 'Y' x 8191;
Lines 872-878 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
872
974
873
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
975
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
874
976
875
    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
977
    $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
876
978
877
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
979
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
878
    is(
980
    is(
Lines 928-934 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
928
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1030
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
929
1031
930
    my $marc_record_1 = MARC::Record->new();
1032
    my $marc_record_1 = MARC::Record->new();
931
    $marc_record_1->leader('     cam  22      a 4500');
1033
    $marc_record_1->leader('     cam a22      a 4500');
932
    $marc_record_1->append_fields(
1034
    $marc_record_1->append_fields(
933
        MARC::Field->new( '001', '123' ),
1035
        MARC::Field->new( '001', '123' ),
934
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
1036
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
Lines 939-945 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
939
        MARC::Field->new( '999', '', '', c => '1234567' ),
1041
        MARC::Field->new( '999', '', '', c => '1234567' ),
940
    );
1042
    );
941
    my $marc_record_2 = MARC::Record->new();
1043
    my $marc_record_2 = MARC::Record->new();
942
    $marc_record_2->leader('     cam  22      a 4500');
1044
    $marc_record_2->leader('     cam a22      a 4500');
943
    $marc_record_2->append_fields(
1045
    $marc_record_2->append_fields(
944
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
1046
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
945
1047
Lines 961-967 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
961
1063
962
    is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' );
1064
    is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' );
963
1065
964
    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
1066
    my $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
965
1067
966
    ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" );
1068
    ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" );
967
    is(
1069
    is(
Lines 1148-1153 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1148
        MARC::Field->new( 150, '', '', a => 'Foo' ),
1250
        MARC::Field->new( 150, '', '', a => 'Foo' ),
1149
        MARC::Field->new( 450, '', '', a => 'Bar' ),
1251
        MARC::Field->new( 450, '', '', a => 'Bar' ),
1150
    );
1252
    );
1253
    $authority_record->encoding('UTF-8');
1254
1151
    $dbh->do(
1255
    $dbh->do(
1152
        "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef,
1256
        "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef,
1153
        ( $authority_record->as_xml_record('MARC21') )
1257
        ( $authority_record->as_xml_record('MARC21') )
Lines 1193-1199 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1193
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1297
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1194
1298
1195
    my $marc_record_1 = MARC::Record->new();
1299
    my $marc_record_1 = MARC::Record->new();
1196
    $marc_record_1->leader('     cam  22      a 4500');
1300
    $marc_record_1->leader('     cam a22      a 4500');
1197
    $marc_record_1->append_fields(
1301
    $marc_record_1->append_fields(
1198
        MARC::Field->new( '001', '123' ),
1302
        MARC::Field->new( '001', '123' ),
1199
        MARC::Field->new( '245', '', '', a => 'Title' ),
1303
        MARC::Field->new( '245', '', '', a => 'Title' ),
1200
- 

Return to bug 27859