View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/BackgroundJob.pm (+1 lines)
Lines 446-451 sub core_types_to_classes { Link Here
446
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
446
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
447
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
447
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
448
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
448
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
449
        search_result_export                => 'Koha::BackgroundJob::SearchResultExport',
449
    };
450
    };
450
}
451
}
451
452
(-)a/Koha/BackgroundJob/SearchResultExport.pm (+186 lines)
Line 0 Link Here
1
package Koha::BackgroundJob::SearchResultExport;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
use Try::Tiny;
20
use Koha::SearchEngine::Search;
21
22
use File::Spec;
23
use File::Path qw(mkpath);
24
use Koha::Email;
25
use Koha::UploadedFiles;
26
use POSIX qw(strftime);
27
use Digest::MD5 qw(md5_hex);
28
use Carp qw(croak);
29
use Encode qw(encode);
30
31
use base 'Koha::BackgroundJob';
32
33
=head1 NAME
34
35
Koha::BackgroundJob::SearchResultExport - Export data from search result
36
37
This is a subclass of Koha::BackgroundJob.
38
39
=head1 API
40
41
=head2 Class methods
42
43
=head3 job_type
44
45
Define the job type of this job: stage_marc_for_import
46
47
=cut
48
49
sub job_type {
50
    return 'search_result_export';
51
}
52
53
=head3 process
54
55
Perform the export of search records.
56
57
=cut
58
59
sub process {
60
    my ( $self, $args ) = @_;
61
62
    $self->start;
63
64
    my $data = $self->decoded_data;
65
    my $borrowernumber = $data->{borrowernumber};
66
    my $elasticsearch_query = $args->{elasticsearch_query};
67
    my $preferred_format = $args->{preferred_format};
68
    my $searcher = Koha::SearchEngine::Search->new({
69
        index => $Koha::SearchEngine::BIBLIOS_INDEX
70
    });
71
    my $elasticsearch = $searcher->get_elasticsearch();
72
73
    my $results = eval {
74
        $elasticsearch->search(
75
            index => $searcher->index_name,
76
            scroll => '1m', #TODO: Syspref for scroll time limit?
77
            size => 1000,  #TODO: Syspref for batch size?
78
            body => $elasticsearch_query
79
        );
80
    };
81
    my @errors;
82
    push @errors, $@ if $@;
83
84
    my @docs;
85
    my $encoded_results;
86
    my %export_links;
87
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
88
89
    if (!@errors) {
90
        my $scroll_id = $results->{_scroll_id};
91
        while (@{$results->{hits}->{hits}}) {
92
            push @docs, @{$results->{hits}->{hits}};
93
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
94
            $results = $elasticsearch->scroll(
95
                scroll => '1m',
96
                scroll_id => $scroll_id
97
            );
98
        }
99
100
        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
101
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
102
        }
103
        else {
104
            $encoded_results->{$preferred_format->{name}} =
105
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
106
        }
107
108
        my %format_extensions = (
109
            'ISO2709' => '.mrc',
110
            'MARCXML' => '.xml',
111
        );
112
113
        my $upload_dir = Koha::UploadedFile->permanent_directory;
114
115
        while (my ($format, $data) = each %{$encoded_results}) {
116
            my $hash = md5_hex($data);
117
            my $category = "search_marc_export";
118
            my $time = strftime "%Y%m%d_%H%M", localtime time;
119
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
120
            my $filename = $category . '_' . $time . $ext;
121
            my $file_dir = File::Spec->catfile($upload_dir, $category);
122
            if ( !-d $file_dir) {
123
                unless(mkpath $file_dir) {
124
                    push @errors, "Failed to create $file_dir";
125
                    next;
126
                }
127
            }
128
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
129
130
            my $fh = IO::File->new($filepath, "w");
131
132
            if ($fh) {
133
                $fh->binmode;
134
                print $fh encode('UTF-8', $data);
135
                $fh->close;
136
137
                my $size = -s $filepath;
138
                my $file = Koha::UploadedFile->new({
139
                        hashvalue => $hash,
140
                        filename  => $filename,
141
                        dir       => $category,
142
                        filesize  => $size,
143
                        owner     => $borrowernumber,
144
                        uploadcategorycode => 'search_marc_export',
145
                        public    => 0,
146
                        permanent => 1,
147
                    })->store;
148
                my $id = $file->_result()->get_column('id');
149
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
150
            }
151
            else {
152
                push @errors, "Failed to write \"$filepath\"";
153
            }
154
        }
155
    }
156
    my $report = {
157
        export_links => \%export_links,
158
        total => scalar @docs,
159
        errors => \@errors,
160
        query_string => $query_string,
161
    };
162
    $data->{report}   = $report;
163
    if (@errors) {
164
        $self->set({ progress => 0, status => 'failed' })->store;
165
    }
166
    else {
167
        $self->finish($data);
168
    }
169
}
170
171
=head3 enqueue
172
173
Enqueue the new job
174
175
=cut
176
177
sub enqueue {
178
    my ( $self, $args) = @_;
179
    $self->SUPER::enqueue({
180
        job_size => $args->{size},
181
        job_args => $args,
182
        job_queue => 'long_tasks',
183
    });
184
}
185
186
1;
(-)a/Koha/SearchEngine/Elasticsearch.pm (-48 / +317 lines)
Lines 43-50 use YAML::XS; Link Here
43
43
44
use List::Util qw( sum0 );
44
use List::Util qw( sum0 );
45
use MARC::File::XML;
45
use MARC::File::XML;
46
use MIME::Base64 qw( encode_base64 );
46
use MIME::Base64 qw(encode_base64 decode_base64);
47
use Encode       qw( encode );
47
use Encode qw(encode decode);
48
48
use Business::ISBN;
49
use Business::ISBN;
49
use Scalar::Util qw( looks_like_number );
50
use Scalar::Util qw( looks_like_number );
50
51
Lines 612-618 sub marc_records_to_documents { Link Here
612
    my $control_fields_rules = $rules->{control_fields};
613
    my $control_fields_rules = $rules->{control_fields};
613
    my $data_fields_rules    = $rules->{data_fields};
614
    my $data_fields_rules    = $rules->{data_fields};
614
    my $marcflavour          = lc C4::Context->preference('marcflavour');
615
    my $marcflavour          = lc C4::Context->preference('marcflavour');
615
    my $use_array            = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
616
616
617
    my @record_documents;
617
    my @record_documents;
618
618
Lines 875-922 sub marc_records_to_documents { Link Here
875
            }
875
            }
876
        }
876
        }
877
877
878
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
878
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
879
        $record->encoding('UTF-8');
879
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
880
        if ($use_array) {
880
            $record,
881
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
881
            $preferred_format,
882
            $record_document->{'marc_format'}     = 'ARRAY';
882
            $marcflavour
883
        } else {
883
        );
884
            my @warnings;
885
            {
886
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
887
                local $SIG{__WARN__} = sub {
888
                    push @warnings, $_[0];
889
                };
890
                my $usmarc_record = $record->as_usmarc();
891
892
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
893
                my $decoded_usmarc_record;
894
                eval { $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); };
895
                if ( $@ || $decoded_usmarc_record->warnings() ) {
896
897
                    #NOTE: We override the warnings since they're many and misleading
898
                    @warnings = (
899
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
900
                    );
901
                }
902
903
                my $marc_data = encode_base64( encode( 'UTF-8', $usmarc_record ) );
904
                $record_document->{'marc_data'} = $marc_data;
905
            }
906
            if (@warnings) {
907
884
908
                # Suppress warnings if record length exceeded
885
        if ($preferred_format eq 'ARRAY') {
909
                unless ( substr( $record->leader(), 0, 5 ) eq '99999' ) {
886
            $record_document->{'marc_data_array'} = $encoded_record;
910
                    foreach my $warning (@warnings) {
887
        } else {
911
                        carp $warning;
888
            $record_document->{'marc_data'} = $encoded_record;
912
                    }
913
                }
914
                $record_document->{'marc_data'}   = $record->as_xml_record($marcflavour);
915
                $record_document->{'marc_format'} = 'MARCXML';
916
            } else {
917
                $record_document->{'marc_format'} = 'base64ISO2709';
918
            }
919
        }
889
        }
890
        $record_document->{'marc_format'} = $format;
920
891
921
        # Check if there is at least one available item
892
        # Check if there is at least one available item
922
        if ( $self->index eq $BIBLIOS_INDEX ) {
893
        if ( $self->index eq $BIBLIOS_INDEX ) {
Lines 941-946 sub marc_records_to_documents { Link Here
941
    return \@record_documents;
912
    return \@record_documents;
942
}
913
}
943
914
915
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
916
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
917
918
Encode a MARC::Record to the preferred marc document record format. If record
919
exceeds ISO2709 maximum size record size and C<$format> is set to
920
'base64ISO2709' format will fallback to 'MARCXML' instead.
921
922
=over 4
923
924
=item C<$record>
925
926
A MARC::Record object
927
928
=item C<$marcflavour>
929
930
The marcflavour to use
931
932
=back
933
934
=cut
935
936
sub search_document_marc_record_encode {
937
    my ($self, $record, $format, $marcflavour) = @_;
938
939
    $record->encoding('UTF-8');
940
941
    if ($format eq 'ARRAY') {
942
        return ($self->_marc_to_array($record), $format);
943
    }
944
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
945
        my @warnings;
946
        my $marc_data;
947
        # Save origial leader since as_usmarc will modify leader
948
        # resulting in failed tests when comparing records
949
        my $original_leader = $record->leader();
950
        {
951
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
952
            local $SIG{__WARN__} = sub {
953
                push @warnings, $_[0];
954
            };
955
            $marc_data = $record->as_usmarc();
956
            if (!@warnings) {
957
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
958
                my $usmarc_record;
959
                eval { $usmarc_record = MARC::Record->new_from_usmarc($marc_data); };
960
                @warnings = $usmarc_record->warnings() if defined $usmarc_record;
961
                if ($@ || @warnings) {
962
                    #NOTE: We override the warnings since they're many and misleading
963
                    @warnings = (
964
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
965
                    );
966
                }
967
            }
968
        }
969
        if (@warnings) {
970
            # Suppress warnings if record length exceeded
971
            unless (substr($record->leader(), 0, 5) eq '99999') {
972
                foreach my $warning (@warnings) {
973
                    carp $warning;
974
                }
975
            }
976
            # Restore original leader
977
            $record->leader($original_leader);
978
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
979
        }
980
        else {
981
            $marc_data = encode('UTF-8', $marc_data);
982
            if ($format eq 'base64ISO2709') {
983
                $marc_data = encode_base64($marc_data);
984
            }
985
            return ($marc_data, $format);
986
        }
987
    }
988
    elsif ($format eq 'MARCXML') {
989
        return (MARC::File::XML::record($record, $marcflavour), $format);
990
    }
991
    else {
992
        # This should be unlikely to happen
993
        croak "Invalid marc record serialization format: $format";
994
    }
995
}
996
997
=head2 search_document_marc_record_decode
998
    my $marc_record = $self->search_document_marc_record_decode(@result);
999
1000
Extract marc data from Elasticsearch result and decode to MARC::Record object
1001
1002
=cut
1003
1004
sub search_document_marc_record_decode {
1005
    # Result is passed in as array, will get flattened
1006
    # and first element will be $result
1007
    my ($self, $result) = @_;
1008
    if ($result->{marc_format} eq 'base64ISO2709') {
1009
        my $marc_data = decode('utf-8', decode_base64($result->{marc_data}));
1010
        my $record = MARC::Record->new_from_usmarc($marc_data);
1011
        return $record;
1012
    }
1013
    elsif ($result->{marc_format} eq 'MARCXML') {
1014
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
1015
    }
1016
    elsif ($result->{marc_format} eq 'ARRAY') {
1017
        return $self->_array_to_marc($result->{marc_data_array});
1018
    }
1019
    else {
1020
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
1021
    }
1022
}
1023
1024
=head2 search_documents_encode($docs, $preferred_format)
1025
1026
    $records_data = $self->search_documents_encode($docs, $preferred_format)
1027
1028
Return marc encoded records from ElasticSearch search result documents. The return value
1029
C<$marc_records> is a hashref with encoded records keyed by MARC format.
1030
1031
=over 4
1032
1033
=item C<$docs>
1034
1035
An arrayref of Elasticsearch search documents
1036
1037
=item C<$preferred_format>
1038
1039
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
1040
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
1041
is set to 'ISO2709'.
1042
1043
=back
1044
1045
=cut
1046
1047
sub search_documents_encode {
1048
1049
    my ($self, $docs, $preferred_format) = @_;
1050
1051
    my %encoded_records = (
1052
        'ISO2709' => [],
1053
        'MARCXML' => []
1054
    );
1055
1056
    unless (exists $encoded_records{$preferred_format}) {
1057
       croak "Invalid preferred format: $preferred_format";
1058
    }
1059
1060
    for my $es_record (@{$docs}) {
1061
        # Special optimized cases
1062
        my $marc_data;
1063
        my $resulting_format = $preferred_format;
1064
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
1065
            $marc_data = $es_record->{_source}{marc_data};
1066
        }
1067
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
1068
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
1069
        }
1070
        else {
1071
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
1072
            my $marcflavour = lc C4::Context->preference('marcflavour');
1073
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
1074
        }
1075
        push @{$encoded_records{$resulting_format}}, $marc_data;
1076
    }
1077
    if (@{$encoded_records{'ISO2709'}}) {
1078
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
1079
    }
1080
    else {
1081
        delete $encoded_records{'ISO2709'};
1082
    }
1083
1084
    if (@{$encoded_records{'MARCXML'}}) {
1085
        $encoded_records{'MARCXML'} = join(
1086
            "\n",
1087
            MARC::File::XML::header(),
1088
            join("\n", @{$encoded_records{'MARCXML'}}),
1089
            MARC::File::XML::footer()
1090
        );
1091
    }
1092
    else {
1093
        delete $encoded_records{'MARCXML'};
1094
    }
1095
1096
    return \%encoded_records;
1097
}
1098
1099
=head2 search_result_export_custom_formats()
1100
1101
    $custom_formats = $self->search_result_export_custom_formats()
1102
1103
Return user defined custom search result export formats.
1104
1105
=cut
1106
1107
sub search_result_export_custom_formats {
1108
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
1109
    my $custom_export_formats = {};
1110
1111
    if (ref $export_custom_formats_pref eq 'ARRAY') {
1112
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
1113
            # TODO: Perhaps validate on save or trow error here instead of just
1114
            # ignoring invalid formats
1115
            my $format = $export_custom_formats_pref->[$i];
1116
            if (
1117
                ref $format->{fields} eq 'ARRAY' &&
1118
                @{$format->{fields}} &&
1119
                $format->{name}
1120
            ) {
1121
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
1122
                $custom_export_formats->{"custom_$i"} = $format;
1123
            }
1124
        }
1125
    }
1126
    return $custom_export_formats;
1127
}
1128
1129
=head2 search_documents_custom_format_encode($docs, $custom_format)
1130
1131
    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
1132
1133
Return encoded records from ElasticSearch search result documents using a
1134
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
1135
Returns the encoded records.
1136
1137
=over 4
1138
1139
=item C<$docs>
1140
1141
An arrayref of Elasticsearch search documents
1142
1143
=item C<$format>
1144
1145
A hashref with the custom format definition.
1146
1147
=back
1148
1149
=cut
1150
1151
sub search_documents_custom_format_encode {
1152
    my ($self, $docs, $format) = @_;
1153
1154
    my $result;
1155
1156
    my $doc_get_fields = sub {
1157
        my ($doc, $fields) = @_;
1158
        my @row;
1159
        foreach my $field (@{$fields}) {
1160
            my $values = $doc->{_source}->{$field};
1161
            push @row, ref $values eq 'ARRAY' ? $values : [''];
1162
        }
1163
        return \@row;
1164
    };
1165
1166
    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1167
1168
    if($format->{multiple} eq 'ignore') {
1169
        for (my $i = 0; $i < @rows; ++$i) {
1170
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1171
        }
1172
    }
1173
    elsif($format->{multiple} eq 'newline') {
1174
        if (@{$format->{fields}} == 1) {
1175
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
1176
        }
1177
        else {
1178
            croak "'newline' is only valid for single field export formats";
1179
        }
1180
    }
1181
    elsif($format->{multiple} eq 'join') {
1182
        for (my $i = 0; $i < @rows; ++$i) {
1183
            # Escape separator
1184
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1185
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1186
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
1187
                }
1188
            }
1189
            # Separate multiple values with "|"
1190
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
1191
        }
1192
    }
1193
    else {
1194
        croak "Invalid 'multiple' option: " . $format->{multiple};
1195
    }
1196
    if (@{$format->{fields}} == 1) {
1197
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
1198
    }
1199
    else {
1200
        # Encode CSV
1201
        for (my $i = 0; $i < @rows; ++$i) {
1202
            # Escape quotes
1203
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1204
                $rows[$i][$j] =~ s/"/""/g;
1205
            }
1206
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1207
        }
1208
    }
1209
1210
    return join("\n", @rows);
1211
}
1212
944
=head2 _marc_to_array($record)
1213
=head2 _marc_to_array($record)
945
1214
946
    my @fields = _marc_to_array($record)
1215
    my @fields = _marc_to_array($record)
Lines 1012-1029 sub _array_to_marc { Link Here
1012
    $record->leader( $data->{leader} );
1281
    $record->leader( $data->{leader} );
1013
    for my $field ( @{ $data->{fields} } ) {
1282
    for my $field ( @{ $data->{fields} } ) {
1014
        my $tag = ( keys %{$field} )[0];
1283
        my $tag = ( keys %{$field} )[0];
1015
        $field = $field->{$tag};
1284
        my $field_data = $field->{$tag};
1016
        my $marc_field;
1285
        my $marc_field;
1017
        if ( ref($field) eq 'HASH' ) {
1286
        if ( ref($field_data) eq 'HASH' ) {
1018
            my @subfields;
1287
            my @subfields;
1019
            foreach my $subfield ( @{ $field->{subfields} } ) {
1288
            foreach my $subfield ( @{ $field_data->{subfields} } ) {
1020
                my $code = ( keys %{$subfield} )[0];
1289
                my $code = ( keys %{$subfield} )[0];
1021
                push @subfields, $code;
1290
                push @subfields, $code;
1022
                push @subfields, $subfield->{$code};
1291
                push @subfields, $subfield->{$code};
1023
            }
1292
            }
1024
            $marc_field = MARC::Field->new( $tag, $field->{ind1}, $field->{ind2}, @subfields );
1293
            $marc_field = MARC::Field->new( $tag, $field_data->{ind1}, $field_data->{ind2}, @subfields );
1025
        } else {
1294
        } else {
1026
            $marc_field = MARC::Field->new( $tag, $field );
1295
            $marc_field = MARC::Field->new( $tag, $field_data );
1027
        }
1296
        }
1028
        $record->append_fields($marc_field);
1297
        $record->append_fields($marc_field);
1029
    }
1298
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-26 / +3 lines)
Lines 177-183 sub search_compat { Link Here
177
    my $index = $offset;
177
    my $index = $offset;
178
    my $hits  = $results->{'hits'};
178
    my $hits  = $results->{'hits'};
179
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
179
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
180
        $records[ $index++ ] = $self->decode_record_from_result( $es_record->{'_source'} );
180
        $records[ $index++ ] = $self->search_document_marc_record_decode( $es_record->{'_source'} );
181
    }
181
    }
182
182
183
    # consumers of this expect a name-spaced result, we provide the default
183
    # consumers of this expect a name-spaced result, we provide the default
Lines 245-251 sub search_auth_compat { Link Here
245
            # it's not reproduced here yet.
245
            # it's not reproduced here yet.
246
            my $authtype           = $rs->single;
246
            my $authtype           = $rs->single;
247
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
247
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
248
            my $marc               = $self->decode_record_from_result($record);
248
            my $marc               = $self->search_document_marc_record_decode($record);
249
            my $mainentry          = $marc->field($auth_tag_to_report);
249
            my $mainentry          = $marc->field($auth_tag_to_report);
250
            my $reported_tag;
250
            my $reported_tag;
251
            if ($mainentry) {
251
            if ($mainentry) {
Lines 384-390 sub simple_search_compat { Link Here
384
    my @records;
384
    my @records;
385
    my $hits = $results->{'hits'};
385
    my $hits = $results->{'hits'};
386
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
386
    foreach my $es_record ( @{ $hits->{'hits'} } ) {
387
        push @records, $self->decode_record_from_result( $es_record->{'_source'} );
387
        push @records, $self->search_document_marc_record_decode( $es_record->{'_source'} );
388
    }
388
    }
389
    return ( undef, \@records, $hits->{'total'} );
389
    return ( undef, \@records, $hits->{'total'} );
390
}
390
}
Lines 404-432 sub extract_biblionumber { Link Here
404
    return Koha::SearchEngine::Search::extract_biblionumber($searchresultrecord);
404
    return Koha::SearchEngine::Search::extract_biblionumber($searchresultrecord);
405
}
405
}
406
406
407
=head2 decode_record_from_result
408
    my $marc_record = $self->decode_record_from_result(@result);
409
410
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
411
412
=cut
413
414
sub decode_record_from_result {
415
416
    # Result is passed in as array, will get flattened
417
    # and first element will be $result
418
    my ( $self, $result ) = @_;
419
    if ( $result->{marc_format} eq 'base64ISO2709' ) {
420
        return MARC::Record->new_from_usmarc( decode_base64( $result->{marc_data} ) );
421
    } elsif ( $result->{marc_format} eq 'MARCXML' ) {
422
        return MARC::Record->new_from_xml( $result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour') );
423
    } elsif ( $result->{marc_format} eq 'ARRAY' ) {
424
        return $self->_array_to_marc( $result->{marc_data_array} );
425
    } else {
426
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
427
    }
428
}
429
430
=head2 max_result_window
407
=head2 max_result_window
431
408
432
Returns the maximum number of results that can be fetched
409
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+45 lines)
Lines 148-153 use C4::Languages qw( getlanguage getLanguages ); Link Here
148
use C4::Koha        qw( getitemtypeimagelocation GetAuthorisedValues );
148
use C4::Koha        qw( getitemtypeimagelocation GetAuthorisedValues );
149
use URI::Escape;
149
use URI::Escape;
150
use POSIX qw(ceil floor);
150
use POSIX qw(ceil floor);
151
use Carp qw(croak);
151
152
152
use Koha::ItemTypes;
153
use Koha::ItemTypes;
153
use Koha::Library::Groups;
154
use Koha::Library::Groups;
Lines 157-162 use Koha::SearchEngine::QueryBuilder; Link Here
157
use Koha::Virtualshelves;
158
use Koha::Virtualshelves;
158
use Koha::SearchFields;
159
use Koha::SearchFields;
159
use Koha::SearchFilters;
160
use Koha::SearchFilters;
161
use Koha::BackgroundJob::SearchResultExport;
160
162
161
use URI::Escape;
163
use URI::Escape;
162
use JSON qw( decode_json encode_json );
164
use JSON qw( decode_json encode_json );
Lines 757-762 $template->param( Link Here
757
    add_to_some_public_shelves  => $some_public_shelves,
759
    add_to_some_public_shelves  => $some_public_shelves,
758
);
760
);
759
761
762
my $patron = Koha::Patrons->find( $borrowernumber );
763
my $export_enabled =
764
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
765
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
766
    $patron && $patron->has_permission({ tools => 'export_catalog' });
767
768
$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
769
770
if ($export_enabled) {
771
772
    my $export = $cgi->param('export');
773
    my $preferred_format = $cgi->param('export_format');
774
    my $custom_export_formats = $searcher->search_result_export_custom_formats;
775
776
    $template->param(custom_export_formats => $custom_export_formats);
777
778
    # TODO: Need to handle $hits = 0?
779
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
780
781
    if ($export && $preferred_format && $hits) {
782
        unless (
783
            $preferred_format eq 'ISO2709' ||
784
            $preferred_format eq 'MARCXML'
785
        ) {
786
            if (!exists $custom_export_formats->{$preferred_format}) {
787
                croak "Invalid export format: $preferred_format";
788
            }
789
            else {
790
                $preferred_format = $custom_export_formats->{$preferred_format};
791
            }
792
        }
793
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
794
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
795
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
796
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
797
            size => $size,
798
            preferred_format => $preferred_format,
799
            elasticsearch_query => \%export_query
800
        });
801
        $template->param(export_job_id => $export_job_id);
802
    }
803
}
804
760
output_html_with_http_headers $cgi, $cookie, $template->output;
805
output_html_with_http_headers $cgi, $cookie, $template->output;
761
806
762
=head2 prepare_adv_search_types
807
=head2 prepare_adv_search_types
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+19 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
11
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
12
13
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
14
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
15
16
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
17
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
18
    },
19
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (+3 lines)
Lines 246-251 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
246
('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),
246
('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),
247
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
247
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
248
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
248
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
249
('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
249
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
250
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
250
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
251
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
251
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
252
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
Lines 706-711 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
706
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
707
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
707
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
708
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
708
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
709
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
710
('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
711
('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer'),
709
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
712
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
710
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
713
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
711
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
714
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
Line 0 Link Here
1
[% USE Koha %]
2
3
[% BLOCK report %]
4
    [% SET report = job.report %]
5
    [% IF report %]
6
        [% IF job.status == 'finished' %]
7
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
8
            <ul>
9
              <li>[% report.total | html %] records exported</li>
10
              [% IF report.export_links.keys.size > 1 %]
11
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
12
              [% END %]
13
              [% FOREACH format IN report.export_links.keys.sort %]
14
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
15
              [% END %]
16
            </ul>
17
        [% ELSE %]
18
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
19
            <ul>
20
            [% FOREACH error in report.errors %]
21
              <li>[% error | html %]</li>
22
            [% END %]
23
            </ul>
24
        [% END %]
25
    [% END %]
26
[% END %]
27
28
[% BLOCK detail %]
29
[% END %]
30
31
[% BLOCK js %]
32
[% END %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
Lines 227-232 Link Here
227
                '_id': 'import_from_kbart_file',
227
                '_id': 'import_from_kbart_file',
228
                '_str': _("Import titles from a KBART file")
228
                '_str': _("Import titles from a KBART file")
229
            },
229
            },
230
            {
231
                '_id': 'search_result_export',
232
                '_str': _("Search result export")
233
            },
230
        ];
234
        ];
231
235
232
        function get_job_type (job_type) {
236
        function get_job_type (job_type) {
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
Lines 167-172 Searching: Link Here
167
                  1: use
167
                  1: use
168
                  0: "don't use"
168
                  0: "don't use"
169
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
169
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
170
        -
171
            - pref: EnableElasticsearchSearchResultExport
172
              type: boolean
173
              default: yes
174
              choices:
175
                  1: Enable
176
                  0: Disable
177
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
178
        -
179
            - pref: ElasticsearchSearchResultExportCustomFormats
180
              type: textarea
181
              syntax: text/x-yaml
182
              class: code
183
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
184
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
185
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
186
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
187
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
188
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
189
            - <p>The behavior when handling fields with multiple values.</p>
190
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
191
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
192
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
193
            - 'Example:</br>'
194
            - '- name: Biblionumbers<br />'
195
            - '&nbsp;&nbsp;fields: [local-number]<br />'
196
            - '&nbsp;&nbsp;multiple: ignore<br />'
197
            - '- name: Title and author<br />'
198
            - '&nbsp;&nbsp;fields: [title, author]<br />'
199
            - '&nbsp;&nbsp;multiple: join<br /><br />'
200
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
201
        -
202
            - Limit export from search results to a maximum of
203
            - pref: ElasticsearchSearchResultExportLimit
204
              class: integer
205
            - search result items (Elasticsearch only).<br /><br />
206
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
170
    Results display:
207
    Results display:
171
        -
208
        -
172
            - pref: numSearchResultsDropdown
209
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+52 lines)
Lines 362-367 Link Here
362
                    </div>
362
                    </div>
363
                    <!-- /.btn-group -->
363
                    <!-- /.btn-group -->
364
364
365
                    [% IF export_enabled %]
366
                        <div class="btn-group">
367
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
368
                                Export all results<span class="caret"></span>
369
                            </button>
370
                            <ul class="dropdown-menu">
371
                                <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
372
                                <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
373
                                [% FOREACH id IN custom_export_formats.keys %]
374
                                    <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id | uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
375
                                [% END %]
376
                           </ul>
377
                        </div> <!-- /.btn-group -->
378
                    [% END %]
379
365
                    [% IF Koha.Preference('numSearchResultsDropdown') %]
380
                    [% IF Koha.Preference('numSearchResultsDropdown') %]
366
                        <div class="btn-group">
381
                        <div class="btn-group">
367
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Results per page: [% results_per_page | html %] </button>
382
                            <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Results per page: [% results_per_page | html %] </button>
Lines 451-456 Link Here
451
        >
466
        >
452
    [% END %]
467
    [% END %]
453
468
469
    [% IF export_job_id %]
470
        <div class="dialog message">
471
          <p>Exporting records, the export will be processed as soon as possible.</p>
472
           [% INCLUDE "job_progress.inc" job_id=export_job_id %]
473
          <p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id | uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
474
          <div id="job_callback"></div>
475
        </div>
476
    [% END %]
477
478
454
    <!-- Search Results Table -->
479
    <!-- Search Results Table -->
455
    [% IF ( total ) %]
480
    [% IF ( total ) %]
456
        [% IF ( scan ) %]
481
        [% IF ( scan ) %]
Lines 922-928 Link Here
922
    [% Asset.css("css/humanmsg.css") | $raw %]
947
    [% Asset.css("css/humanmsg.css") | $raw %]
923
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
948
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
924
    [% INCLUDE 'select2.inc' %]
949
    [% INCLUDE 'select2.inc' %]
950
    [% INCLUDE 'str/job_progress.inc' %]
951
    [% Asset.js("js/job_progress.js") | $raw %]
925
    <script>
952
    <script>
953
        [% IF export_job_id %]
954
            updateProgress([% export_job_id | html %], function() {
955
                $.getJSON('/api/v1/jobs/[% export_job_id | html %]', function(job) {
956
                    if (job.data.report.errors.length) {
957
                        humanMsg.displayMsg(
958
                            _("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
959
                            { className: 'humanError' }
960
                        );
961
                    }
962
                    else {
963
                        let export_links = Object.entries(job.data.report.export_links);
964
                        let export_links_html = export_links.map(([format, href]) =>
965
                            `<p>${format}: <a href=${href}>${href}</a></p>`
966
                        ).join('');
967
                        if (export_links.length > 1) {
968
                            export_links_html =
969
                                `<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
970
                        }
971
                        $(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
972
                            .appendTo("#job_callback");
973
                    }
974
                });
975
            });
976
        [% END %]
977
926
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
978
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
927
        var q_array = new Array();  // will hold search terms, if present
979
        var q_array = new Array();  // will hold search terms, if present
928
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
980
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-21 / +124 lines)
Lines 16-22 Link Here
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
use Encode;
19
use utf8;
20
20
21
use Test::More tests => 8;
21
use Test::More tests => 8;
22
use Test::Exception;
22
use Test::Exception;
Lines 209-215 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
209
209
210
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
210
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
211
211
212
    plan tests => 70;
212
    plan tests => 81;
213
213
214
    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
214
    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
215
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );
215
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );
Lines 426-431 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
426
            marc_type   => 'marc21',
426
            marc_type   => 'marc21',
427
            marc_field  => '522a',
427
            marc_field  => '522a',
428
        },
428
        },
429
        {
430
            name => 'local-number',
431
            type => 'string',
432
            facet => 0,
433
            suggestible => 0,
434
            searchable => 1,
435
            sort => 1,
436
            marc_type => 'marc21',
437
            marc_field => '999c',
438
        },
429
    );
439
    );
430
440
431
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
441
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 458-464 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
458
    my $long_callno = '1234567890' x 30;
468
    my $long_callno = '1234567890' x 30;
459
469
460
    my $marc_record_1 = MARC::Record->new();
470
    my $marc_record_1 = MARC::Record->new();
461
    $marc_record_1->leader('     cam  22      a 4500');
471
    $marc_record_1->leader('     cam a22      a 4500');
462
    $marc_record_1->append_fields(
472
    $marc_record_1->append_fields(
463
        MARC::Field->new( '001', '123' ),
473
        MARC::Field->new( '001', '123' ),
464
        MARC::Field->new( '007', 'ku' ),
474
        MARC::Field->new( '007', 'ku' ),
Lines 479-486 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
479
        MARC::Field->new( '952', '', '', 0 => 0,    g => '127.20', o => $callno2,     l => 2 ),
489
        MARC::Field->new( '952', '', '', 0 => 0,    g => '127.20', o => $callno2,     l => 2 ),
480
        MARC::Field->new( '952', '', '', 0 => 1,    g => '0.00',   o => $long_callno, l => 1 ),
490
        MARC::Field->new( '952', '', '', 0 => 1,    g => '0.00',   o => $long_callno, l => 1 ),
481
    );
491
    );
492
482
    my $marc_record_2 = MARC::Record->new();
493
    my $marc_record_2 = MARC::Record->new();
483
    $marc_record_2->leader('     cam  22      a 4500');
494
    $marc_record_2->leader('     cam a22      a 4500');
484
    $marc_record_2->append_fields(
495
    $marc_record_2->append_fields(
485
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
496
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
486
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
497
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
Lines 493-499 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
493
    );
504
    );
494
505
495
    my $marc_record_3 = MARC::Record->new();
506
    my $marc_record_3 = MARC::Record->new();
496
    $marc_record_3->leader('     cam  22      a 4500');
507
    $marc_record_3->leader('     cam a22      a 4500');
497
    $marc_record_3->append_fields(
508
    $marc_record_3->append_fields(
498
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
509
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
499
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
510
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
Lines 506-512 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
506
    );
517
    );
507
518
508
    my $marc_record_4 = MARC::Record->new();
519
    my $marc_record_4 = MARC::Record->new();
509
    $marc_record_4->leader('     cam  22      a 4500');
520
    $marc_record_4->leader('     cam a22      a 4500');
510
    $marc_record_4->append_fields(
521
    $marc_record_4->append_fields(
511
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
522
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
512
        MARC::Field->new( '100', '', '',  a => 'Author 2' ),
523
        MARC::Field->new( '100', '', '',  a => 'Author 2' ),
Lines 628-634 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
628
    ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' );
639
    ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' );
629
    is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' );
640
    is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' );
630
641
631
    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
642
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
632
643
633
    ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" );
644
    ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" );
634
    is(
645
    is(
Lines 747-766 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
747
    # Marc serialization format fallback for records exceeding base64ISO2709 max record size
758
    # Marc serialization format fallback for records exceeding base64ISO2709 max record size
748
759
749
    my $large_marc_record = MARC::Record->new();
760
    my $large_marc_record = MARC::Record->new();
750
    $large_marc_record->leader('     cam  22      a 4500');
761
    $large_marc_record->leader('     cam a22      a 4500');
751
762
752
    $large_marc_record->append_fields(
763
    $large_marc_record->append_fields(
753
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
764
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
754
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
765
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
755
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
766
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
756
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large record' ),
767
        # "|" is for testing escaping for multiple values with custom format
757
        MARC::Field->new( '999', '', '', c => '1234567' ),
768
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large | record' ),
769
        MARC::Field->new( '999', '', '', c => '1234569' ),
758
    );
770
    );
759
771
760
    my $item_field = MARC::Field->new(
772
    my $item_field = MARC::Field->new(
761
        '952', '', '', o => '123456789123456789123456789', p => '123456789',
773
        '952', '', '', o => '123456789123456789123456789', p => '123456789',
762
        z => Encode::decode( 'UTF-8', 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.' )
774
        z => 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.'
763
    );
775
    );
776
764
    my $items_count = 1638;
777
    my $items_count = 1638;
765
    while ( --$items_count ) {
778
    while ( --$items_count ) {
766
        $large_marc_record->append_fields($item_field);
779
        $large_marc_record->append_fields($item_field);
Lines 773-779 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
773
        'For record exceeding max record size marc_format should be set correctly'
786
        'For record exceeding max record size marc_format should be set correctly'
774
    );
787
    );
775
788
776
    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
789
    $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
777
790
778
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
791
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
779
    is(
792
    is(
Lines 781-786 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
781
        "Decoded MARCXML record has same data as original record"
794
        "Decoded MARCXML record has same data as original record"
782
    );
795
    );
783
796
797
    # Search export functionality
798
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
799
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
800
    my @es_response_docs;
801
    my $records_data;
802
803
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
804
805
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
806
807
        $docs = $see->marc_records_to_documents(\@source_docs);
808
809
        # Emulate Elasticsearch response docs structure
810
        @es_response_docs = map { { _source => $_ } } @{$docs};
811
812
        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
813
814
        # $large_marc_record should not have been encoded as ISO2709
815
        # since exceeds maximum size, see above
816
        my @tmp = ($marc_record_1, $marc_record_2);
817
        is(
818
            $records_data->{ISO2709},
819
            join('', map { $_->as_usmarc() } @tmp),
820
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
821
        );
822
823
        my $expected_marc_xml = join("\n",
824
            MARC::File::XML::header(),
825
            MARC::File::XML::record($large_marc_record, 'MARC21'),
826
            MARC::File::XML::footer()
827
        );
828
829
        is(
830
            $records_data->{MARCXML},
831
            $expected_marc_xml,
832
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
833
        );
834
835
        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
836
837
        $expected_marc_xml = join("\n",
838
            MARC::File::XML::header(),
839
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
840
            MARC::File::XML::footer()
841
        );
842
843
        is(
844
            $records_data->{MARCXML},
845
            $expected_marc_xml,
846
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
847
        );
848
    }
849
850
    my $custom_formats = <<'END';
851
- name: Biblionumbers
852
  fields: [local-number]
853
  multiple: ignore
854
- name: Title and author
855
  fields: [title, author]
856
  multiple: join
857
END
858
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
859
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
860
861
    # Biblionumbers custom format
862
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
863
    # UTF-8 encode?
864
    is(
865
        $records_data,
866
        "1234567\n1234568\n1234569",
867
        "Records where correctly encoded for the custom format \"Biblionumbers\""
868
    );
869
870
    # Title and author custom format
871
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
872
873
    my $encoded_data = join(
874
        "\n",
875
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
876
        "\"\",\"Author 2\"",
877
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
878
    );
879
880
    is(
881
        $records_data,
882
        $encoded_data,
883
        "Records where correctly encoded for the custom format \"Title and author\""
884
    );
885
784
    push @mappings, {
886
    push @mappings, {
785
        name        => 'title',
887
        name        => 'title',
786
        type        => 'string',
888
        type        => 'string',
Lines 824-830 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
824
926
825
    pop @mappings;
927
    pop @mappings;
826
    my $marc_record_with_blank_field = MARC::Record->new();
928
    my $marc_record_with_blank_field = MARC::Record->new();
827
    $marc_record_with_blank_field->leader('     cam  22      a 4500');
929
    $marc_record_with_blank_field->leader('     cam a22      a 4500');
828
930
829
    $marc_record_with_blank_field->append_fields(
931
    $marc_record_with_blank_field->append_fields(
830
        MARC::Field->new( '100', '', '', a => '' ),
932
        MARC::Field->new( '100', '', '', a => '' ),
Lines 837-843 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
837
    is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' );
939
    is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' );
838
940
839
    my $marc_record_with_large_field = MARC::Record->new();
941
    my $marc_record_with_large_field = MARC::Record->new();
840
    $marc_record_with_large_field->leader('     cam  22      a 4500');
942
    $marc_record_with_large_field->leader('     cam a22      a 4500');
841
943
842
    my $xs = 'X' x 8191;
944
    my $xs = 'X' x 8191;
843
    my $ys = 'Y' x 8191;
945
    my $ys = 'Y' x 8191;
Lines 867-873 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
867
969
868
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
970
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
869
971
870
    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
972
    $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
871
973
872
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
974
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
873
    is(
975
    is(
Lines 923-929 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
923
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1025
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
924
1026
925
    my $marc_record_1 = MARC::Record->new();
1027
    my $marc_record_1 = MARC::Record->new();
926
    $marc_record_1->leader('     cam  22      a 4500');
1028
    $marc_record_1->leader('     cam a22      a 4500');
927
    $marc_record_1->append_fields(
1029
    $marc_record_1->append_fields(
928
        MARC::Field->new( '001', '123' ),
1030
        MARC::Field->new( '001', '123' ),
929
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
1031
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
Lines 934-940 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
934
        MARC::Field->new( '999', '', '', c => '1234567' ),
1036
        MARC::Field->new( '999', '', '', c => '1234567' ),
935
    );
1037
    );
936
    my $marc_record_2 = MARC::Record->new();
1038
    my $marc_record_2 = MARC::Record->new();
937
    $marc_record_2->leader('     cam  22      a 4500');
1039
    $marc_record_2->leader('     cam a22      a 4500');
938
    $marc_record_2->append_fields(
1040
    $marc_record_2->append_fields(
939
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
1041
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
940
1042
Lines 956-962 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
956
1058
957
    is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' );
1059
    is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' );
958
1060
959
    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
1061
    my $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
960
1062
961
    ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" );
1063
    ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" );
962
    is(
1064
    is(
Lines 1091-1096 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1091
        MARC::Field->new( 150, '', '', a => 'Foo' ),
1193
        MARC::Field->new( 150, '', '', a => 'Foo' ),
1092
        MARC::Field->new( 450, '', '', a => 'Bar' ),
1194
        MARC::Field->new( 450, '', '', a => 'Bar' ),
1093
    );
1195
    );
1196
    $authority_record->encoding('UTF-8');
1197
1094
    $dbh->do(
1198
    $dbh->do(
1095
        "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef,
1199
        "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef,
1096
        ( $authority_record->as_xml_record('MARC21') )
1200
        ( $authority_record->as_xml_record('MARC21') )
Lines 1136-1142 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1136
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1240
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
1137
1241
1138
    my $marc_record_1 = MARC::Record->new();
1242
    my $marc_record_1 = MARC::Record->new();
1139
    $marc_record_1->leader('     cam  22      a 4500');
1243
    $marc_record_1->leader('     cam a22      a 4500');
1140
    $marc_record_1->append_fields(
1244
    $marc_record_1->append_fields(
1141
        MARC::Field->new( '001', '123' ),
1245
        MARC::Field->new( '001', '123' ),
1142
        MARC::Field->new( '245', '', '', a => 'Title' ),
1246
        MARC::Field->new( '245', '', '', a => 'Title' ),
Lines 1174-1180 subtest 'marc_records_to_documents should set the "available" field' => sub { Link Here
1174
    $see->get_elasticsearch_mappings();
1278
    $see->get_elasticsearch_mappings();
1175
1279
1176
    my $marc_record_1 = MARC::Record->new();
1280
    my $marc_record_1 = MARC::Record->new();
1177
    $marc_record_1->leader('     cam  22      a 4500');
1281
    $marc_record_1->leader('     cam a22      a 4500');
1178
    $marc_record_1->append_fields(
1282
    $marc_record_1->append_fields(
1179
        MARC::Field->new( '245', '', '', a => 'Title' ),
1283
        MARC::Field->new( '245', '', '', a => 'Title' ),
1180
    );
1284
    );
1181
- 

Return to bug 27859