View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/BackgroundJob.pm (+1 lines)
Lines 447-452 sub core_types_to_classes { Link Here
447
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
447
        marc_import_revert_batch            => 'Koha::BackgroundJob::MARCImportRevertBatch',
448
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
448
        pseudonymize_statistic              => 'Koha::BackgroundJob::PseudonymizeStatistic',
449
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
449
        import_from_kbart_file              => 'Koha::BackgroundJob::ImportKBARTFile',
450
        search_result_export                => 'Koha::BackgroundJob::SearchResultExport',
450
    };
451
    };
451
}
452
}
452
453
(-)a/Koha/BackgroundJob/SearchResultExport.pm (+186 lines)
Line 0 Link Here
1
package Koha::BackgroundJob::SearchResultExport;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
use Try::Tiny;
20
use Koha::SearchEngine::Search;
21
22
use File::Spec;
23
use File::Path qw(mkpath);
24
use Koha::Email;
25
use Koha::UploadedFiles;
26
use POSIX qw(strftime);
27
use Digest::MD5 qw(md5_hex);
28
use Carp qw(croak);
29
use Encode qw(encode);
30
31
use base 'Koha::BackgroundJob';
32
33
=head1 NAME
34
35
Koha::BackgroundJob::SearchResultExport - Export data from search result
36
37
This is a subclass of Koha::BackgroundJob.
38
39
=head1 API
40
41
=head2 Class methods
42
43
=head3 job_type
44
45
Define the job type of this job: stage_marc_for_import
46
47
=cut
48
49
sub job_type {
50
    return 'search_result_export';
51
}
52
53
=head3 process
54
55
Perform the export of search records.
56
57
=cut
58
59
sub process {
60
    my ( $self, $args ) = @_;
61
62
    $self->start;
63
64
    my $data = $self->decoded_data;
65
    my $borrowernumber = $data->{borrowernumber};
66
    my $elasticsearch_query = $args->{elasticsearch_query};
67
    my $preferred_format = $args->{preferred_format};
68
    my $searcher = Koha::SearchEngine::Search->new({
69
        index => $Koha::SearchEngine::BIBLIOS_INDEX
70
    });
71
    my $elasticsearch = $searcher->get_elasticsearch();
72
73
    my $results = eval {
74
        $elasticsearch->search(
75
            index => $searcher->index_name,
76
            scroll => '1m', #TODO: Syspref for scroll time limit?
77
            size => 1000,  #TODO: Syspref for batch size?
78
            body => $elasticsearch_query
79
        );
80
    };
81
    my @errors;
82
    push @errors, $@ if $@;
83
84
    my @docs;
85
    my $encoded_results;
86
    my %export_links;
87
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
88
89
    if (!@errors) {
90
        my $scroll_id = $results->{_scroll_id};
91
        while (@{$results->{hits}->{hits}}) {
92
            push @docs, @{$results->{hits}->{hits}};
93
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
94
            $results = $elasticsearch->scroll(
95
                scroll => '1m',
96
                scroll_id => $scroll_id
97
            );
98
        }
99
100
        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
101
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
102
        }
103
        else {
104
            $encoded_results->{$preferred_format->{name}} =
105
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
106
        }
107
108
        my %format_extensions = (
109
            'ISO2709' => '.mrc',
110
            'MARCXML' => '.xml',
111
        );
112
113
        my $upload_dir = Koha::UploadedFile->permanent_directory;
114
115
        while (my ($format, $data) = each %{$encoded_results}) {
116
            my $hash = md5_hex($data);
117
            my $category = "search_marc_export";
118
            my $time = strftime "%Y%m%d_%H%M", localtime time;
119
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
120
            my $filename = $category . '_' . $time . $ext;
121
            my $file_dir = File::Spec->catfile($upload_dir, $category);
122
            if ( !-d $file_dir) {
123
                unless(mkpath $file_dir) {
124
                    push @errors, "Failed to create $file_dir";
125
                    next;
126
                }
127
            }
128
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
129
130
            my $fh = IO::File->new($filepath, "w");
131
132
            if ($fh) {
133
                $fh->binmode;
134
                print $fh encode('UTF-8', $data);
135
                $fh->close;
136
137
                my $size = -s $filepath;
138
                my $file = Koha::UploadedFile->new({
139
                        hashvalue => $hash,
140
                        filename  => $filename,
141
                        dir       => $category,
142
                        filesize  => $size,
143
                        owner     => $borrowernumber,
144
                        uploadcategorycode => 'search_marc_export',
145
                        public    => 0,
146
                        permanent => 1,
147
                    })->store;
148
                my $id = $file->_result()->get_column('id');
149
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
150
            }
151
            else {
152
                push @errors, "Failed to write \"$filepath\"";
153
            }
154
        }
155
    }
156
    my $report = {
157
        export_links => \%export_links,
158
        total => scalar @docs,
159
        errors => \@errors,
160
        query_string => $query_string,
161
    };
162
    $data->{report}   = $report;
163
    if (@errors) {
164
        $self->set({ progress => 0, status => 'failed' })->store;
165
    }
166
    else {
167
        $self->finish($data);
168
    }
169
}
170
171
=head3 enqueue
172
173
Enqueue the new job
174
175
=cut
176
177
sub enqueue {
178
    my ( $self, $args) = @_;
179
    $self->SUPER::enqueue({
180
        job_size => $args->{size},
181
        job_args => $args,
182
        job_queue => 'long_tasks',
183
    });
184
}
185
186
1;
(-)a/Koha/SearchEngine/Elasticsearch.pm (-49 / +316 lines)
Lines 43-50 use YAML::XS; Link Here
43
43
44
use List::Util qw( sum0 );
44
use List::Util qw( sum0 );
45
use MARC::File::XML;
45
use MARC::File::XML;
46
use MIME::Base64 qw( encode_base64 );
46
use MIME::Base64 qw(encode_base64 decode_base64);
47
use Encode qw( encode );
47
use Encode qw(encode decode);
48
use Business::ISBN;
48
use Business::ISBN;
49
use Scalar::Util qw( looks_like_number );
49
use Scalar::Util qw( looks_like_number );
50
50
Lines 588-594 sub marc_records_to_documents { Link Here
588
    my $control_fields_rules = $rules->{control_fields};
588
    my $control_fields_rules = $rules->{control_fields};
589
    my $data_fields_rules = $rules->{data_fields};
589
    my $data_fields_rules = $rules->{data_fields};
590
    my $marcflavour = lc C4::Context->preference('marcflavour');
590
    my $marcflavour = lc C4::Context->preference('marcflavour');
591
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
592
591
593
    my @record_documents;
592
    my @record_documents;
594
593
Lines 826-874 sub marc_records_to_documents { Link Here
826
                }
825
                }
827
            }
826
            }
828
        }
827
        }
828
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
829
829
830
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
830
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
831
        $record->encoding('UTF-8');
831
            $record,
832
        if ($use_array) {
832
            $preferred_format,
833
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
833
            $marcflavour
834
            $record_document->{'marc_format'} = 'ARRAY';
834
        );
835
        } else {
836
            my @warnings;
837
            {
838
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
839
                local $SIG{__WARN__} = sub {
840
                    push @warnings, $_[0];
841
                };
842
                my $usmarc_record = $record->as_usmarc();
843
844
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
845
                my $decoded_usmarc_record;
846
                eval { $decoded_usmarc_record = MARC::Record->new_from_usmarc($usmarc_record); };
847
                if ( $@ || $decoded_usmarc_record->warnings() ) {
848
849
                    #NOTE: We override the warnings since they're many and misleading
850
                    @warnings = (
851
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
852
                    );
853
                }
854
835
855
                my $marc_data = encode_base64( encode( 'UTF-8', $usmarc_record ) );
836
        if ($preferred_format eq 'ARRAY') {
856
                $record_document->{'marc_data'} = $marc_data;
837
            $record_document->{'marc_data_array'} = $encoded_record;
857
            }
838
        } else {
858
            if (@warnings) {
839
            $record_document->{'marc_data'} = $encoded_record;
859
                # Suppress warnings if record length exceeded
860
                unless (substr($record->leader(), 0, 5) eq '99999') {
861
                    foreach my $warning (@warnings) {
862
                        carp $warning;
863
                    }
864
                }
865
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
866
                $record_document->{'marc_format'} = 'MARCXML';
867
            }
868
            else {
869
                $record_document->{'marc_format'} = 'base64ISO2709';
870
            }
871
        }
840
        }
841
        $record_document->{'marc_format'} = $format;
872
842
873
        # Check if there is at least one available item
843
        # Check if there is at least one available item
874
        if ($self->index eq $BIBLIOS_INDEX) {
844
        if ($self->index eq $BIBLIOS_INDEX) {
Lines 881-887 sub marc_records_to_documents { Link Here
881
                    onloan       => undef,
851
                    onloan       => undef,
882
                    itemlost     => 0,
852
                    itemlost     => 0,
883
                })->count;
853
                })->count;
884
885
                $record_document->{available} = $avail_items ? \1 : \0;
854
                $record_document->{available} = $avail_items ? \1 : \0;
886
            }
855
            }
887
        }
856
        }
Lines 891-896 sub marc_records_to_documents { Link Here
891
    return \@record_documents;
860
    return \@record_documents;
892
}
861
}
893
862
863
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
864
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
865
866
Encode a MARC::Record to the preferred marc document record format. If record
867
exceeds ISO2709 maximum size record size and C<$format> is set to
868
'base64ISO2709' format will fallback to 'MARCXML' instead.
869
870
=over 4
871
872
=item C<$record>
873
874
A MARC::Record object
875
876
=item C<$marcflavour>
877
878
The marcflavour to use
879
880
=back
881
882
=cut
883
884
sub search_document_marc_record_encode {
885
    my ($self, $record, $format, $marcflavour) = @_;
886
887
    $record->encoding('UTF-8');
888
889
    if ($format eq 'ARRAY') {
890
        return ($self->_marc_to_array($record), $format);
891
    }
892
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
893
        my @warnings;
894
        my $marc_data;
895
        # Save origial leader since as_usmarc will modify leader
896
        # resulting in failed tests when comparing records
897
        my $original_leader = $record->leader();
898
        {
899
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
900
            local $SIG{__WARN__} = sub {
901
                push @warnings, $_[0];
902
            };
903
            $marc_data = $record->as_usmarc();
904
            if (!@warnings) {
905
                #NOTE: Try to round-trip the record to prove it will work for retrieval after searching
906
                my $usmarc_record;
907
                eval { $usmarc_record = MARC::Record->new_from_usmarc($marc_data); };
908
                @warnings = $usmarc_record->warnings() if defined $usmarc_record;
909
                if ($@ || @warnings) {
910
                    #NOTE: We override the warnings since they're many and misleading
911
                    @warnings = (
912
                        "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.",
913
                    );
914
                }
915
            }
916
        }
917
        if (@warnings) {
918
            # Suppress warnings if record length exceeded
919
            unless (substr($record->leader(), 0, 5) eq '99999') {
920
                foreach my $warning (@warnings) {
921
                    carp $warning;
922
                }
923
            }
924
            # Restore original leader
925
            $record->leader($original_leader);
926
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
927
        }
928
        else {
929
            $marc_data = encode('UTF-8', $marc_data);
930
            if ($format eq 'base64ISO2709') {
931
                $marc_data = encode_base64($marc_data);
932
            }
933
            return ($marc_data, $format);
934
        }
935
    }
936
    elsif ($format eq 'MARCXML') {
937
        return (MARC::File::XML::record($record, $marcflavour), $format);
938
    }
939
    else {
940
        # This should be unlikely to happen
941
        croak "Invalid marc record serialization format: $format";
942
    }
943
}
944
945
=head2 search_document_marc_record_decode
946
    my $marc_record = $self->search_document_marc_record_decode(@result);
947
948
Extract marc data from Elasticsearch result and decode to MARC::Record object
949
950
=cut
951
952
sub search_document_marc_record_decode {
953
    # Result is passed in as array, will get flattened
954
    # and first element will be $result
955
    my ($self, $result) = @_;
956
    if ($result->{marc_format} eq 'base64ISO2709') {
957
        my $marc_data = decode('utf-8', decode_base64($result->{marc_data}));
958
        my $record = MARC::Record->new_from_usmarc($marc_data);
959
        return $record;
960
    }
961
    elsif ($result->{marc_format} eq 'MARCXML') {
962
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
963
    }
964
    elsif ($result->{marc_format} eq 'ARRAY') {
965
        return $self->_array_to_marc($result->{marc_data_array});
966
    }
967
    else {
968
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
969
    }
970
}
971
972
=head2 search_documents_encode($docs, $preferred_format)
973
974
    $records_data = $self->search_documents_encode($docs, $preferred_format)
975
976
Return marc encoded records from ElasticSearch search result documents. The return value
977
C<$marc_records> is a hashref with encoded records keyed by MARC format.
978
979
=over 4
980
981
=item C<$docs>
982
983
An arrayref of Elasticsearch search documents
984
985
=item C<$preferred_format>
986
987
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
988
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
989
is set to 'ISO2709'.
990
991
=back
992
993
=cut
994
995
sub search_documents_encode {
996
997
    my ($self, $docs, $preferred_format) = @_;
998
999
    my %encoded_records = (
1000
        'ISO2709' => [],
1001
        'MARCXML' => []
1002
    );
1003
1004
    unless (exists $encoded_records{$preferred_format}) {
1005
       croak "Invalid preferred format: $preferred_format";
1006
    }
1007
1008
    for my $es_record (@{$docs}) {
1009
        # Special optimized cases
1010
        my $marc_data;
1011
        my $resulting_format = $preferred_format;
1012
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
1013
            $marc_data = $es_record->{_source}{marc_data};
1014
        }
1015
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
1016
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
1017
        }
1018
        else {
1019
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
1020
            my $marcflavour = lc C4::Context->preference('marcflavour');
1021
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
1022
        }
1023
        push @{$encoded_records{$resulting_format}}, $marc_data;
1024
    }
1025
    if (@{$encoded_records{'ISO2709'}}) {
1026
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
1027
    }
1028
    else {
1029
        delete $encoded_records{'ISO2709'};
1030
    }
1031
1032
    if (@{$encoded_records{'MARCXML'}}) {
1033
        $encoded_records{'MARCXML'} = join(
1034
            "\n",
1035
            MARC::File::XML::header(),
1036
            join("\n", @{$encoded_records{'MARCXML'}}),
1037
            MARC::File::XML::footer()
1038
        );
1039
    }
1040
    else {
1041
        delete $encoded_records{'MARCXML'};
1042
    }
1043
1044
    return \%encoded_records;
1045
}
1046
1047
=head2 search_result_export_custom_formats()
1048
1049
    $custom_formats = $self->search_result_export_custom_formats()
1050
1051
Return user defined custom search result export formats.
1052
1053
=cut
1054
1055
sub search_result_export_custom_formats {
1056
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
1057
    my $custom_export_formats = {};
1058
1059
    if (ref $export_custom_formats_pref eq 'ARRAY') {
1060
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
1061
            # TODO: Perhaps validate on save or trow error here instead of just
1062
            # ignoring invalid formats
1063
            my $format = $export_custom_formats_pref->[$i];
1064
            if (
1065
                ref $format->{fields} eq 'ARRAY' &&
1066
                @{$format->{fields}} &&
1067
                $format->{name}
1068
            ) {
1069
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
1070
                $custom_export_formats->{"custom_$i"} = $format;
1071
            }
1072
        }
1073
    }
1074
    return $custom_export_formats;
1075
}
1076
1077
=head2 search_documents_custom_format_encode($docs, $custom_format)
1078
1079
    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
1080
1081
Return encoded records from ElasticSearch search result documents using a
1082
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
1083
Returns the encoded records.
1084
1085
=over 4
1086
1087
=item C<$docs>
1088
1089
An arrayref of Elasticsearch search documents
1090
1091
=item C<$format>
1092
1093
A hashref with the custom format definition.
1094
1095
=back
1096
1097
=cut
1098
1099
sub search_documents_custom_format_encode {
1100
    my ($self, $docs, $format) = @_;
1101
1102
    my $result;
1103
1104
    my $doc_get_fields = sub {
1105
        my ($doc, $fields) = @_;
1106
        my @row;
1107
        foreach my $field (@{$fields}) {
1108
            my $values = $doc->{_source}->{$field};
1109
            push @row, ref $values eq 'ARRAY' ? $values : [''];
1110
        }
1111
        return \@row;
1112
    };
1113
1114
    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1115
1116
    if($format->{multiple} eq 'ignore') {
1117
        for (my $i = 0; $i < @rows; ++$i) {
1118
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1119
        }
1120
    }
1121
    elsif($format->{multiple} eq 'newline') {
1122
        if (@{$format->{fields}} == 1) {
1123
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
1124
        }
1125
        else {
1126
            croak "'newline' is only valid for single field export formats";
1127
        }
1128
    }
1129
    elsif($format->{multiple} eq 'join') {
1130
        for (my $i = 0; $i < @rows; ++$i) {
1131
            # Escape separator
1132
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1133
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1134
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
1135
                }
1136
            }
1137
            # Separate multiple values with "|"
1138
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
1139
        }
1140
    }
1141
    else {
1142
        croak "Invalid 'multiple' option: " . $format->{multiple};
1143
    }
1144
    if (@{$format->{fields}} == 1) {
1145
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
1146
    }
1147
    else {
1148
        # Encode CSV
1149
        for (my $i = 0; $i < @rows; ++$i) {
1150
            # Escape quotes
1151
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1152
                $rows[$i][$j] =~ s/"/""/g;
1153
            }
1154
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1155
        }
1156
    }
1157
1158
    return encode('UTF-8', join("\n", @rows));
1159
}
1160
894
=head2 _marc_to_array($record)
1161
=head2 _marc_to_array($record)
895
1162
896
    my @fields = _marc_to_array($record)
1163
    my @fields = _marc_to_array($record)
Lines 962-979 sub _array_to_marc { Link Here
962
    $record->leader($data->{leader});
1229
    $record->leader($data->{leader});
963
    for my $field (@{$data->{fields}}) {
1230
    for my $field (@{$data->{fields}}) {
964
        my $tag = (keys %{$field})[0];
1231
        my $tag = (keys %{$field})[0];
965
        $field = $field->{$tag};
1232
        my $field_data = $field->{$tag};
966
        my $marc_field;
1233
        my $marc_field;
967
        if (ref($field) eq 'HASH') {
1234
        if (ref($field_data) eq 'HASH') {
968
            my @subfields;
1235
            my @subfields;
969
            foreach my $subfield (@{$field->{subfields}}) {
1236
            foreach my $subfield (@{$field_data->{subfields}}) {
970
                my $code = (keys %{$subfield})[0];
1237
                my $code = (keys %{$subfield})[0];
971
                push @subfields, $code;
1238
                push @subfields, $code;
972
                push @subfields, $subfield->{$code};
1239
                push @subfields, $subfield->{$code};
973
            }
1240
            }
974
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
1241
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
975
        } else {
1242
        } else {
976
            $marc_field = MARC::Field->new($tag, $field)
1243
            $marc_field = MARC::Field->new($tag, $field_data)
977
        }
1244
        }
978
        $record->append_fields($marc_field);
1245
        $record->append_fields($marc_field);
979
    }
1246
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-28 / +3 lines)
Lines 175-181 sub search_compat { Link Here
175
    my $index = $offset;
175
    my $index = $offset;
176
    my $hits = $results->{'hits'};
176
    my $hits = $results->{'hits'};
177
    foreach my $es_record (@{$hits->{'hits'}}) {
177
    foreach my $es_record (@{$hits->{'hits'}}) {
178
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
178
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
179
    }
179
    }
180
180
181
    # consumers of this expect a name-spaced result, we provide the default
181
    # consumers of this expect a name-spaced result, we provide the default
Lines 242-248 sub search_auth_compat { Link Here
242
            # it's not reproduced here yet.
242
            # it's not reproduced here yet.
243
            my $authtype           = $rs->single;
243
            my $authtype           = $rs->single;
244
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
244
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
245
            my $marc               = $self->decode_record_from_result($record);
245
            my $marc               = $self->search_document_marc_record_decode($record);
246
            my $mainentry          = $marc->field($auth_tag_to_report);
246
            my $mainentry          = $marc->field($auth_tag_to_report);
247
            my $reported_tag;
247
            my $reported_tag;
248
            if ($mainentry) {
248
            if ($mainentry) {
Lines 376-382 sub simple_search_compat { Link Here
376
    my @records;
376
    my @records;
377
    my $hits = $results->{'hits'};
377
    my $hits = $results->{'hits'};
378
    foreach my $es_record (@{$hits->{'hits'}}) {
378
    foreach my $es_record (@{$hits->{'hits'}}) {
379
        push @records, $self->decode_record_from_result($es_record->{'_source'});
379
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
380
    }
380
    }
381
    return (undef, \@records, $hits->{'total'});
381
    return (undef, \@records, $hits->{'total'});
382
}
382
}
Lines 396-426 sub extract_biblionumber { Link Here
396
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
396
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
397
}
397
}
398
398
399
=head2 decode_record_from_result
400
    my $marc_record = $self->decode_record_from_result(@result);
401
402
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
403
404
=cut
405
406
sub decode_record_from_result {
407
    # Result is passed in as array, will get flattened
408
    # and first element will be $result
409
    my ( $self, $result ) = @_;
410
    if ($result->{marc_format} eq 'base64ISO2709') {
411
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
412
    }
413
    elsif ($result->{marc_format} eq 'MARCXML') {
414
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
415
    }
416
    elsif ($result->{marc_format} eq 'ARRAY') {
417
        return $self->_array_to_marc($result->{marc_data_array});
418
    }
419
    else {
420
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
421
    }
422
}
423
424
=head2 max_result_window
399
=head2 max_result_window
425
400
426
Returns the maximum number of results that can be fetched
401
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+45 lines)
Lines 148-153 use C4::Languages qw( getlanguage getLanguages ); Link Here
148
use C4::Koha qw( getitemtypeimagelocation GetAuthorisedValues );
148
use C4::Koha qw( getitemtypeimagelocation GetAuthorisedValues );
149
use URI::Escape;
149
use URI::Escape;
150
use POSIX qw(ceil floor);
150
use POSIX qw(ceil floor);
151
use Carp qw(croak);
151
152
152
use Koha::ItemTypes;
153
use Koha::ItemTypes;
153
use Koha::Library::Groups;
154
use Koha::Library::Groups;
Lines 157-162 use Koha::SearchEngine::QueryBuilder; Link Here
157
use Koha::Virtualshelves;
158
use Koha::Virtualshelves;
158
use Koha::SearchFields;
159
use Koha::SearchFields;
159
use Koha::SearchFilters;
160
use Koha::SearchFilters;
161
use Koha::BackgroundJob::SearchResultExport;
160
162
161
use URI::Escape;
163
use URI::Escape;
162
use JSON qw( decode_json encode_json );
164
use JSON qw( decode_json encode_json );
Lines 737-742 $template->param( Link Here
737
    add_to_some_public_shelves  => $some_public_shelves,
739
    add_to_some_public_shelves  => $some_public_shelves,
738
);
740
);
739
741
742
my $patron = Koha::Patrons->find( $borrowernumber );
743
my $export_enabled =
744
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
745
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
746
    $patron && $patron->has_permission({ tools => 'export_catalog' });
747
748
$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
749
750
if ($export_enabled) {
751
752
    my $export = $cgi->param('export');
753
    my $preferred_format = $cgi->param('export_format');
754
    my $custom_export_formats = $searcher->search_result_export_custom_formats;
755
756
    $template->param(custom_export_formats => $custom_export_formats);
757
758
    # TODO: Need to handle $hits = 0?
759
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
760
761
    if ($export && $preferred_format && $hits) {
762
        unless (
763
            $preferred_format eq 'ISO2709' ||
764
            $preferred_format eq 'MARCXML'
765
        ) {
766
            if (!exists $custom_export_formats->{$preferred_format}) {
767
                croak "Invalid export format: $preferred_format";
768
            }
769
            else {
770
                $preferred_format = $custom_export_formats->{$preferred_format};
771
            }
772
        }
773
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
774
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
775
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
776
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
777
            size => $size,
778
            preferred_format => $preferred_format,
779
            elasticsearch_query => \%export_query
780
        });
781
        $template->param(export_job_id => $export_job_id);
782
    }
783
}
784
740
output_html_with_http_headers $cgi, $cookie, $template->output;
785
output_html_with_http_headers $cgi, $cookie, $template->output;
741
786
742
787
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+23 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
11
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
12
13
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
14
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
15
16
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
17
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
18
19
        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
20
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
21
        say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
22
    },
23
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +4 lines)
Lines 232-238 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
232
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
232
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
233
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
233
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
234
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
234
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
235
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
235
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
236
('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
236
('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
237
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
237
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
238
('EmailFieldPrecedence','email|emailpro|B_email','','Ordered list of patron email fields to use when AutoEmailPrimaryAddress is set to first valid','multiple'),
238
('EmailFieldPrecedence','email|emailpro|B_email','','Ordered list of patron email fields to use when AutoEmailPrimaryAddress is set to first valid','multiple'),
Lines 246-251 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
246
('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),
246
('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),
247
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
247
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
248
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
248
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
249
('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
249
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
250
('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
250
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
251
('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
251
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
252
('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
Lines 705-710 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
705
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
706
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
706
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
707
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
707
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
708
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
709
('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
710
('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer'),
708
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
711
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
709
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
712
('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
710
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
713
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
Line 0 Link Here
1
[% USE Koha %]
2
3
[% BLOCK report %]
4
    [% SET report = job.report %]
5
    [% IF report %]
6
        [% IF job.status == 'finished' %]
7
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
8
            <ul>
9
              <li>[% report.total | html %] records exported</li>
10
              [% IF report.export_links.keys.size > 1 %]
11
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
12
              [% END %]
13
              [% FOREACH format IN report.export_links.keys.sort %]
14
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
15
              [% END %]
16
            </ul>
17
        [% ELSE %]
18
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
19
            <ul>
20
            [% FOREACH error in report.errors %]
21
              <li>[% error | html %]</li>
22
            [% END %]
23
            </ul>
24
        [% END %]
25
    [% END %]
26
[% END %]
27
28
[% BLOCK detail %]
29
[% END %]
30
31
[% BLOCK js %]
32
[% END %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
Lines 238-243 Link Here
238
                '_id': 'import_from_kbart_file',
238
                '_id': 'import_from_kbart_file',
239
                '_str': _("Import titles from a KBART file")
239
                '_str': _("Import titles from a KBART file")
240
            },
240
            },
241
            {
242
                '_id': 'search_result_export',
243
                '_str': _("Search result export")
244
            },
241
        ];
245
        ];
242
246
243
        function get_job_type (job_type) {
247
        function get_job_type (job_type) {
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
Lines 304-312 Administration: Link Here
304
        -
304
        -
305
            - "Elasticsearch MARC format: "
305
            - "Elasticsearch MARC format: "
306
            - pref: ElasticsearchMARCFormat
306
            - pref: ElasticsearchMARCFormat
307
              default: "ISO2709"
307
              default: "base64ISO2709"
308
              choices:
308
              choices:
309
                "ISO2709": "ISO2709 (exchange format)"
309
                "base64ISO2709": "ISO2709 (exchange format)"
310
                "ARRAY": "Searchable array"
310
                "ARRAY": "Searchable array"
311
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
311
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
312
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
312
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
Lines 167-172 Searching: Link Here
167
                  1: use
167
                  1: use
168
                  0: "don't use"
168
                  0: "don't use"
169
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
169
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
170
        -
171
            - pref: EnableElasticsearchSearchResultExport
172
              type: boolean
173
              default: yes
174
              choices:
175
                  1: Enable
176
                  0: Disable
177
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
178
        -
179
            - pref: ElasticsearchSearchResultExportCustomFormats
180
              type: textarea
181
              syntax: text/x-yaml
182
              class: code
183
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
184
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
185
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
186
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
187
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
188
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
189
            - <p>The behavior when handling fields with multiple values.</p>
190
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
191
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
192
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
193
            - 'Example:</br>'
194
            - '- name: Biblionumbers<br />'
195
            - '&nbsp;&nbsp;fields: [local-number]<br />'
196
            - '&nbsp;&nbsp;multiple: ignore<br />'
197
            - '- name: Title and author<br />'
198
            - '&nbsp;&nbsp;fields: [title, author]<br />'
199
            - '&nbsp;&nbsp;multiple: join<br /><br />'
200
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
201
        -
202
            - Limit export from search results to a maximum of
203
            - pref: ElasticsearchSearchResultExportLimit
204
              class: integer
205
            - search result items (Elasticsearch only).<br /><br />
206
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
170
    Results display:
207
    Results display:
171
        -
208
        -
172
            - pref: numSearchResultsDropdown
209
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+51 lines)
Lines 339-344 Link Here
339
                                </div> <!-- /.btn-group -->
339
                                </div> <!-- /.btn-group -->
340
                            [% END %]
340
                            [% END %]
341
341
342
                            [% IF export_enabled %]
343
                                <div class="btn-group">
344
                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
345
                                        Export all results<span class="caret"></span>
346
                                    </button>
347
                                    <ul class="dropdown-menu">
348
                                        <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
349
                                        <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
350
                                        [% FOREACH id IN custom_export_formats.keys %]
351
                                            <li><a class="dropdown-item" href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id | uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
352
                                        [% END %]
353
                                   </ul>
354
                                </div> <!-- /.btn-group -->
355
                            [% END %]
356
342
                        </div> <!-- /#selection_ops -->
357
                        </div> <!-- /#selection_ops -->
343
                        <form id="build_batch_record_modification" method="post" action="/cgi-bin/koha/tools/batch_record_modification.pl">
358
                        <form id="build_batch_record_modification" method="post" action="/cgi-bin/koha/tools/batch_record_modification.pl">
344
                            [% INCLUDE 'csrf-token.inc' %]
359
                            [% INCLUDE 'csrf-token.inc' %]
Lines 380-385 Link Here
380
                    <div class="alert alert-warning"><p><strong>Error:</strong> [% query_error | html %]</p></div>
395
                    <div class="alert alert-warning"><p><strong>Error:</strong> [% query_error | html %]</p></div>
381
                [% END %]
396
                [% END %]
382
397
398
                [% IF export_job_id %]
399
                    <div class="dialog message">
400
                      <p>Exporting records, the export will be processed as soon as possible.</p>
401
                       [% INCLUDE "job_progress.inc" job_id=export_job_id %]
402
                      <p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id | uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
403
                      <div id="job_callback"></div>
404
                    </div>
405
                [% END %]
406
383
                <!-- Search Results Table -->
407
                <!-- Search Results Table -->
384
                [% IF ( total ) %]
408
                [% IF ( total ) %]
385
                    [% IF ( scan ) %]
409
                    [% IF ( scan ) %]
Lines 781-787 Link Here
781
    [% Asset.css("css/humanmsg.css") | $raw %]
805
    [% Asset.css("css/humanmsg.css") | $raw %]
782
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
806
    [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %]
783
    [% INCLUDE 'select2.inc' %]
807
    [% INCLUDE 'select2.inc' %]
808
    [% INCLUDE 'str/job_progress.inc' %]
809
    [% Asset.js("js/job_progress.js") | $raw %]
784
    <script>
810
    <script>
811
        [% IF export_job_id %]
812
            updateProgress([% export_job_id | html %], function() {
813
                $.getJSON('/api/v1/jobs/[% export_job_id | html %]', function(job) {
814
                    if (job.data.report.errors.length) {
815
                        humanMsg.displayMsg(
816
                            _("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
817
                            { className: 'humanError' }
818
                        );
819
                    }
820
                    else {
821
                        let export_links = Object.entries(job.data.report.export_links);
822
                        let export_links_html = export_links.map(([format, href]) =>
823
                            `<p>${format}: <a href=${href}>${href}</a></p>`
824
                        ).join('');
825
                        if (export_links.length > 1) {
826
                            export_links_html =
827
                                `<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
828
                        }
829
                        $(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
830
                            .appendTo("#job_callback");
831
                    }
832
                });
833
            });
834
        [% END %]
835
785
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
836
        var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') | html %]", 10);
786
        var q_array = new Array();  // will hold search terms, if present
837
        var q_array = new Array();  // will hold search terms, if present
787
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
838
        var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') | html %]", 10);
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-23 / +124 lines)
Lines 16-22 Link Here
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
use Encode;
19
use utf8;
20
20
21
use Test::More tests => 8;
21
use Test::More tests => 8;
22
use Test::Exception;
22
use Test::Exception;
Lines 187-196 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
187
187
188
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
188
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
189
189
190
    plan tests => 70;
190
    plan tests => 81;
191
191
192
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
192
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
193
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
193
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
194
194
195
    my @mappings = (
195
    my @mappings = (
196
        {
196
        {
Lines 404-409 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
404
            marc_type   => 'marc21',
404
            marc_type   => 'marc21',
405
            marc_field  => '522a',
405
            marc_field  => '522a',
406
        },
406
        },
407
        {
408
            name => 'local-number',
409
            type => 'string',
410
            facet => 0,
411
            suggestible => 0,
412
            searchable => 1,
413
            sort => 1,
414
            marc_type => 'marc21',
415
            marc_field => '999c',
416
        },
407
    );
417
    );
408
418
409
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
419
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 432-438 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
432
    my $long_callno = '1234567890' x 30;
442
    my $long_callno = '1234567890' x 30;
433
443
434
    my $marc_record_1 = MARC::Record->new();
444
    my $marc_record_1 = MARC::Record->new();
435
    $marc_record_1->leader('     cam  22      a 4500');
445
    $marc_record_1->leader('     cam a22      a 4500');
436
    $marc_record_1->append_fields(
446
    $marc_record_1->append_fields(
437
        MARC::Field->new('001', '123'),
447
        MARC::Field->new('001', '123'),
438
        MARC::Field->new('007', 'ku'),
448
        MARC::Field->new('007', 'ku'),
Lines 452-459 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
452
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
462
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
453
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
463
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
454
    );
464
    );
465
455
    my $marc_record_2 = MARC::Record->new();
466
    my $marc_record_2 = MARC::Record->new();
456
    $marc_record_2->leader('     cam  22      a 4500');
467
    $marc_record_2->leader('     cam a22      a 4500');
457
    $marc_record_2->append_fields(
468
    $marc_record_2->append_fields(
458
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
469
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
459
        MARC::Field->new('100', '', '', a => 'Author 2'),
470
        MARC::Field->new('100', '', '', a => 'Author 2'),
Lines 465-471 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
465
    );
476
    );
466
477
467
    my $marc_record_3 = MARC::Record->new();
478
    my $marc_record_3 = MARC::Record->new();
468
    $marc_record_3->leader('     cam  22      a 4500');
479
    $marc_record_3->leader('     cam a22      a 4500');
469
    $marc_record_3->append_fields(
480
    $marc_record_3->append_fields(
470
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
481
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
471
        MARC::Field->new('100', '', '', a => 'Author 2'),
482
        MARC::Field->new('100', '', '', a => 'Author 2'),
Lines 477-483 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
477
    );
488
    );
478
489
479
    my $marc_record_4 = MARC::Record->new();
490
    my $marc_record_4 = MARC::Record->new();
480
    $marc_record_4->leader('     cam  22      a 4500');
491
    $marc_record_4->leader('     cam a22      a 4500');
481
    $marc_record_4->append_fields(
492
    $marc_record_4->append_fields(
482
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
493
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
483
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
494
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
Lines 580-586 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
580
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
591
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
581
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
592
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
582
593
583
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
594
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
584
595
585
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
596
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
586
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
597
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 681-697 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
681
    # Marc serialization format fallback for records exceeding ISO2709 max record size
692
    # Marc serialization format fallback for records exceeding ISO2709 max record size
682
693
683
    my $large_marc_record = MARC::Record->new();
694
    my $large_marc_record = MARC::Record->new();
684
    $large_marc_record->leader('     cam  22      a 4500');
695
    $large_marc_record->leader('     cam a22      a 4500');
685
696
686
    $large_marc_record->append_fields(
697
    $large_marc_record->append_fields(
687
        MARC::Field->new('100', '', '', a => 'Author 1'),
698
        MARC::Field->new('100', '', '', a => 'Author 1'),
688
        MARC::Field->new('110', '', '', a => 'Corp Author'),
699
        MARC::Field->new('110', '', '', a => 'Corp Author'),
689
        MARC::Field->new('210', '', '', a => 'Title 1'),
700
        MARC::Field->new('210', '', '', a => 'Title 1'),
690
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
701
        # "|" is for testing escaping for multiple values with custom format
691
        MARC::Field->new('999', '', '', c => '1234567'),
702
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large | record'),
703
        MARC::Field->new('999', '', '', c => '1234569'),
692
    );
704
    );
693
705
694
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => Encode::decode('UTF-8','To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.'));
706
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.');
695
    my $items_count = 1638;
707
    my $items_count = 1638;
696
    while(--$items_count) {
708
    while(--$items_count) {
697
        $large_marc_record->append_fields($item_field);
709
        $large_marc_record->append_fields($item_field);
Lines 701-711 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
701
713
702
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
714
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
703
715
704
    $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
716
    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
705
717
706
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
718
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
707
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
719
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
708
720
721
    # Search export functionality
722
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
723
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
724
    my @es_response_docs;
725
    my $records_data;
726
727
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
728
729
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
730
731
        $docs = $see->marc_records_to_documents(\@source_docs);
732
733
        # Emulate Elasticsearch response docs structure
734
        @es_response_docs = map { { _source => $_ } } @{$docs};
735
736
        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
737
738
        # $large_marc_record should not have been encoded as ISO2709
739
        # since exceeds maximum size, see above
740
        my @tmp = ($marc_record_1, $marc_record_2);
741
        is(
742
            $records_data->{ISO2709},
743
            join('', map { $_->as_usmarc() } @tmp),
744
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
745
        );
746
747
        my $expected_marc_xml = join("\n",
748
            MARC::File::XML::header(),
749
            MARC::File::XML::record($large_marc_record, 'MARC21'),
750
            MARC::File::XML::footer()
751
        );
752
753
        is(
754
            $records_data->{MARCXML},
755
            $expected_marc_xml,
756
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
757
        );
758
759
        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
760
761
        $expected_marc_xml = join("\n",
762
            MARC::File::XML::header(),
763
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
764
            MARC::File::XML::footer()
765
        );
766
767
        is(
768
            $records_data->{MARCXML},
769
            $expected_marc_xml,
770
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
771
        );
772
    }
773
774
    my $custom_formats = <<'END';
775
- name: Biblionumbers
776
  fields: [local-number]
777
  multiple: ignore
778
- name: Title and author
779
  fields: [title, author]
780
  multiple: join
781
END
782
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
783
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
784
785
    # Biblionumbers custom format
786
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
787
    # UTF-8 encode?
788
    is(
789
        $records_data,
790
        "1234567\n1234568\n1234569",
791
        "Records where correctly encoded for the custom format \"Biblionumbers\""
792
    );
793
794
    # Title and author custom format
795
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
796
797
    my $encoded_data = join(
798
        "\n",
799
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
800
        "\"\",\"Author 2\"",
801
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
802
    );
803
804
    is(
805
        $records_data,
806
        $encoded_data,
807
        "Records where correctly encoded for the custom format \"Title and author\""
808
    );
809
709
    push @mappings, {
810
    push @mappings, {
710
        name => 'title',
811
        name => 'title',
711
        type => 'string',
812
        type => 'string',
Lines 751-757 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
751
852
752
    pop @mappings;
853
    pop @mappings;
753
    my $marc_record_with_blank_field = MARC::Record->new();
854
    my $marc_record_with_blank_field = MARC::Record->new();
754
    $marc_record_with_blank_field->leader('     cam  22      a 4500');
855
    $marc_record_with_blank_field->leader('     cam a22      a 4500');
755
856
756
    $marc_record_with_blank_field->append_fields(
857
    $marc_record_with_blank_field->append_fields(
757
        MARC::Field->new('100', '', '', a => ''),
858
        MARC::Field->new('100', '', '', a => ''),
Lines 764-770 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
764
    is_deeply( $docs->[0]->{author__suggestion},[],'No value placed into suggestion if mapped marc field is blank');
865
    is_deeply( $docs->[0]->{author__suggestion},[],'No value placed into suggestion if mapped marc field is blank');
765
866
766
    my $marc_record_with_large_field = MARC::Record->new();
867
    my $marc_record_with_large_field = MARC::Record->new();
767
    $marc_record_with_large_field->leader('     cam  22      a 4500');
868
    $marc_record_with_large_field->leader('     cam a22      a 4500');
768
869
769
    my $xs = 'X' x 8191;
870
    my $xs = 'X' x 8191;
770
    my $ys = 'Y' x 8191;
871
    my $ys = 'Y' x 8191;
Lines 794-800 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
794
895
795
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
896
    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );
796
897
797
    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );
898
    $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] );
798
899
799
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
900
    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
800
    is(
901
    is(
Lines 846-852 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
846
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
947
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
847
948
848
    my $marc_record_1 = MARC::Record->new();
949
    my $marc_record_1 = MARC::Record->new();
849
    $marc_record_1->leader('     cam  22      a 4500');
950
    $marc_record_1->leader('     cam a22      a 4500');
850
    $marc_record_1->append_fields(
951
    $marc_record_1->append_fields(
851
        MARC::Field->new('001', '123'),
952
        MARC::Field->new('001', '123'),
852
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
953
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
Lines 857-863 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
857
        MARC::Field->new('999', '', '', c => '1234567'),
958
        MARC::Field->new('999', '', '', c => '1234567'),
858
    );
959
    );
859
    my $marc_record_2 = MARC::Record->new();
960
    my $marc_record_2 = MARC::Record->new();
860
    $marc_record_2->leader('     cam  22      a 4500');
961
    $marc_record_2->leader('     cam a22      a 4500');
861
    $marc_record_2->append_fields(
962
    $marc_record_2->append_fields(
862
        MARC::Field->new('100', '', '', a => 'Author 2'),
963
        MARC::Field->new('100', '', '', a => 'Author 2'),
863
        # MARC::Field->new('210', '', '', a => 'Title 2'),
964
        # MARC::Field->new('210', '', '', a => 'Title 2'),
Lines 878-884 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
878
979
879
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
980
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
880
981
881
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
982
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
882
983
883
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
984
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
884
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
985
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 889-895 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
889
    plan tests => 5;
990
    plan tests => 5;
890
991
891
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
992
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
892
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
993
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
893
994
894
    my $builder = t::lib::TestBuilder->new;
995
    my $builder = t::lib::TestBuilder->new;
895
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
996
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
Lines 1005-1010 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1005
        MARC::Field->new(150, '', '', a => 'Foo'),
1106
        MARC::Field->new(150, '', '', a => 'Foo'),
1006
        MARC::Field->new(450, '', '', a => 'Bar'),
1107
        MARC::Field->new(450, '', '', a => 'Bar'),
1007
    );
1108
    );
1109
    $authority_record->encoding('UTF-8');
1008
    $dbh->do( "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef, ($authority_record->as_xml_record('MARC21') ) );
1110
    $dbh->do( "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef, ($authority_record->as_xml_record('MARC21') ) );
1009
    my $authid = $dbh->last_insert_id( undef, undef, 'auth_header', 'authid' );
1111
    my $authid = $dbh->last_insert_id( undef, undef, 'auth_header', 'authid' );
1010
1112
Lines 1043-1049 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
1043
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
1145
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
1044
1146
1045
    my $marc_record_1 = MARC::Record->new();
1147
    my $marc_record_1 = MARC::Record->new();
1046
    $marc_record_1->leader('     cam  22      a 4500');
1148
    $marc_record_1->leader('     cam a22      a 4500');
1047
    $marc_record_1->append_fields(
1149
    $marc_record_1->append_fields(
1048
        MARC::Field->new('001', '123'),
1150
        MARC::Field->new('001', '123'),
1049
        MARC::Field->new('245', '', '', a => 'Title'),
1151
        MARC::Field->new('245', '', '', a => 'Title'),
Lines 1077-1083 subtest 'marc_records_to_documents should set the "available" field' => sub { Link Here
1077
    $see->get_elasticsearch_mappings();
1179
    $see->get_elasticsearch_mappings();
1078
1180
1079
    my $marc_record_1 = MARC::Record->new();
1181
    my $marc_record_1 = MARC::Record->new();
1080
    $marc_record_1->leader('     cam  22      a 4500');
1182
    $marc_record_1->leader('     cam a22      a 4500');
1081
    $marc_record_1->append_fields(
1183
    $marc_record_1->append_fields(
1082
        MARC::Field->new('245', '', '', a => 'Title'),
1184
        MARC::Field->new('245', '', '', a => 'Title'),
1083
    );
1185
    );
1084
- 

Return to bug 27859