View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-33 / +181 lines)
Lines 41-48 use YAML::XS; Link Here
41
41
42
use List::Util qw( sum0 );
42
use List::Util qw( sum0 );
43
use MARC::File::XML;
43
use MARC::File::XML;
44
use MIME::Base64 qw( encode_base64 );
44
use MIME::Base64 qw(encode_base64 decode_base64);
45
use Encode qw( encode );
45
use Encode qw(encode decode);
46
use Business::ISBN;
46
use Business::ISBN;
47
use Scalar::Util qw( looks_like_number );
47
use Scalar::Util qw( looks_like_number );
48
48
Lines 541-547 sub marc_records_to_documents { Link Here
541
    my $control_fields_rules = $rules->{control_fields};
541
    my $control_fields_rules = $rules->{control_fields};
542
    my $data_fields_rules = $rules->{data_fields};
542
    my $data_fields_rules = $rules->{data_fields};
543
    my $marcflavour = lc C4::Context->preference('marcflavour');
543
    my $marcflavour = lc C4::Context->preference('marcflavour');
544
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
545
544
546
    my @record_documents;
545
    my @record_documents;
547
546
Lines 741-778 sub marc_records_to_documents { Link Here
741
                }
740
                }
742
            }
741
            }
743
        }
742
        }
743
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
744
744
745
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
745
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
746
        $record->encoding('UTF-8');
746
            $record,
747
        if ($use_array) {
747
            $preferred_format,
748
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
748
            $marcflavour
749
            $record_document->{'marc_format'} = 'ARRAY';
749
        );
750
751
        if ($preferred_format eq 'ARRAY') {
752
            $record_document->{'marc_data_array'} = $encoded_record;
750
        } else {
753
        } else {
751
            my @warnings;
754
            $record_document->{'marc_data'} = $encoded_record;
752
            {
755
        }
753
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
756
        $record_document->{'marc_format'} = $format;
754
                local $SIG{__WARN__} = sub {
757
755
                    push @warnings, $_[0];
758
        push @record_documents, $record_document;
756
                };
759
    }
757
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
760
    return \@record_documents;
758
            }
761
}
759
            if (@warnings) {
762
760
                # Suppress warnings if record length exceeded
763
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
761
                unless (substr($record->leader(), 0, 5) eq '99999') {
764
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
762
                    foreach my $warning (@warnings) {
765
763
                        carp $warning;
766
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
764
                    }
767
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.
768
769
=over 4
770
771
=item C<$record>
772
773
A MARC::Record object
774
775
=item C<$marcflavour>
776
777
The marcflavour to use
778
779
=back
780
781
=cut
782
783
sub search_document_marc_record_encode {
784
    my ($self, $record, $format, $marcflavour) = @_;
785
786
    $record->encoding('UTF-8');
787
788
    if ($format eq 'ARRAY') {
789
        return ($self->_marc_to_array($record), $format);
790
    }
791
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
792
        my @warnings;
793
        my $marc_data;
794
        {
795
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
796
            local $SIG{__WARN__} = sub {
797
                push @warnings, $_[0];
798
            };
799
            $marc_data = $record->as_usmarc();
800
        }
801
        if (@warnings) {
802
            # Suppress warnings if record length exceeded
803
            unless (substr($record->leader(), 0, 5) eq '99999') {
804
                foreach my $warning (@warnings) {
805
                    carp $warning;
765
                }
806
                }
766
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
767
                $record_document->{'marc_format'} = 'MARCXML';
768
            }
807
            }
769
            else {
808
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
770
                $record_document->{'marc_format'} = 'base64ISO2709';
809
        }
810
        else {
811
            if ($format eq 'base64ISO2709') {
812
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
771
            }
813
            }
814
            return ($marc_data, $format);
772
        }
815
        }
773
        push @record_documents, $record_document;
774
    }
816
    }
775
    return \@record_documents;
817
    elsif ($format eq 'MARCXML') {
818
        return (MARC::File::XML::record($record, $marcflavour), $format);
819
    }
820
    else {
821
        # This should be unlikely to happen
822
        croak "Invalid marc record serialization format: $format";
823
    }
824
}
825
826
=head2 search_document_marc_record_decode
827
    my $marc_record = $self->search_document_marc_record_decode(@result);
828
829
Extract marc data from Elasticsearch result and decode to MARC::Record object
830
831
=cut
832
833
sub search_document_marc_record_decode {
834
    # Result is passed in as array, will get flattened
835
    # and first element will be $result
836
    my ($self, $result) = @_;
837
    if ($result->{marc_format} eq 'base64ISO2709') {
838
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
839
    }
840
    elsif ($result->{marc_format} eq 'MARCXML') {
841
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
842
    }
843
    elsif ($result->{marc_format} eq 'ARRAY') {
844
        return $self->_array_to_marc($result->{marc_data_array});
845
    }
846
    else {
847
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
848
    }
849
}
850
851
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)
852
853
    $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)
854
855
Return marc encoded records from ElasticSearch search result documents. The return value
856
C<$marc_records> is a hashref with encoded records keyed by MARC format.
857
858
=over 4
859
860
=item C<$docs>
861
862
An arrayref of Elasticsearch search documents
863
864
=item C<$preferred_format>
865
866
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
867
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
868
is set to 'ISO2709'.
869
870
=back
871
872
=cut
873
874
sub search_document_marc_records_encode_from_docs {
875
876
    my ($self, $docs, $preferred_format) = @_;
877
878
    my %encoded_records = (
879
        'ISO2709' => [],
880
        'MARCXML' => []
881
    );
882
883
    unless (exists $encoded_records{$preferred_format}) {
884
       croak "Invalid preferred format: $preferred_format";
885
    }
886
887
    for my $es_record (@{$docs}) {
888
        # Special optimized cases
889
        my $marc_data;
890
        my $resulting_format = $preferred_format;
891
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
892
            $marc_data = $es_record->{_source}{marc_data};
893
        }
894
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
895
            # Stored as UTF-8 encoded binary in index, so needs to be decoded
896
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
897
        }
898
        else {
899
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
900
            my $marcflavour = lc C4::Context->preference('marcflavour');
901
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
902
        }
903
        push @{$encoded_records{$resulting_format}}, $marc_data;
904
    }
905
    if (@{$encoded_records{'ISO2709'}}) {
906
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
907
    }
908
    else {
909
        delete $encoded_records{'ISO2709'};
910
    }
911
912
    if (@{$encoded_records{'MARCXML'}}) {
913
        $encoded_records{'MARCXML'} = join("\n",
914
            MARC::File::XML::header(),
915
            join("\n", @{$encoded_records{'MARCXML'}}),
916
            MARC::File::XML::footer()
917
        );
918
    }
919
    else {
920
        delete $encoded_records{'MARCXML'};
921
    }
922
923
    return \%encoded_records;
776
}
924
}
777
925
778
=head2 _marc_to_array($record)
926
=head2 _marc_to_array($record)
Lines 846-863 sub _array_to_marc { Link Here
846
    $record->leader($data->{leader});
994
    $record->leader($data->{leader});
847
    for my $field (@{$data->{fields}}) {
995
    for my $field (@{$data->{fields}}) {
848
        my $tag = (keys %{$field})[0];
996
        my $tag = (keys %{$field})[0];
849
        $field = $field->{$tag};
997
        my $field_data = $field->{$tag};
850
        my $marc_field;
998
        my $marc_field;
851
        if (ref($field) eq 'HASH') {
999
        if (ref($field_data) eq 'HASH') {
852
            my @subfields;
1000
            my @subfields;
853
            foreach my $subfield (@{$field->{subfields}}) {
1001
            foreach my $subfield (@{$field_data->{subfields}}) {
854
                my $code = (keys %{$subfield})[0];
1002
                my $code = (keys %{$subfield})[0];
855
                push @subfields, $code;
1003
                push @subfields, $code;
856
                push @subfields, $subfield->{$code};
1004
                push @subfields, $subfield->{$code};
857
            }
1005
            }
858
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
1006
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
859
        } else {
1007
        } else {
860
            $marc_field = MARC::Field->new($tag, $field)
1008
            $marc_field = MARC::Field->new($tag, $field_data)
861
        }
1009
        }
862
        $record->append_fields($marc_field);
1010
        $record->append_fields($marc_field);
863
    }
1011
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-29 / +3 lines)
Lines 50-56 use Koha::SearchEngine::Search; Link Here
50
use Koha::Exceptions::Elasticsearch;
50
use Koha::Exceptions::Elasticsearch;
51
use MARC::Record;
51
use MARC::Record;
52
use MARC::File::XML;
52
use MARC::File::XML;
53
use MIME::Base64 qw( decode_base64 );
54
53
55
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
54
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
56
55
Lines 164-170 sub search_compat { Link Here
164
    my $index = $offset;
163
    my $index = $offset;
165
    my $hits = $results->{'hits'};
164
    my $hits = $results->{'hits'};
166
    foreach my $es_record (@{$hits->{'hits'}}) {
165
    foreach my $es_record (@{$hits->{'hits'}}) {
167
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
166
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
168
    }
167
    }
169
168
170
    # consumers of this expect a name-spaced result, we provide the default
169
    # consumers of this expect a name-spaced result, we provide the default
Lines 225-231 sub search_auth_compat { Link Here
225
            # it's not reproduced here yet.
224
            # it's not reproduced here yet.
226
            my $authtype           = $rs->single;
225
            my $authtype           = $rs->single;
227
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
226
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
228
            my $marc               = $self->decode_record_from_result($record);
227
            my $marc               = $self->search_document_marc_record_decode($record);
229
            my $mainentry          = $marc->field($auth_tag_to_report);
228
            my $mainentry          = $marc->field($auth_tag_to_report);
230
            my $reported_tag;
229
            my $reported_tag;
231
            if ($mainentry) {
230
            if ($mainentry) {
Lines 345-351 sub simple_search_compat { Link Here
345
    my @records;
344
    my @records;
346
    my $hits = $results->{'hits'};
345
    my $hits = $results->{'hits'};
347
    foreach my $es_record (@{$hits->{'hits'}}) {
346
    foreach my $es_record (@{$hits->{'hits'}}) {
348
        push @records, $self->decode_record_from_result($es_record->{'_source'});
347
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
349
    }
348
    }
350
    return (undef, \@records, $hits->{'total'});
349
    return (undef, \@records, $hits->{'total'});
351
}
350
}
Lines 365-395 sub extract_biblionumber { Link Here
365
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
364
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
366
}
365
}
367
366
368
=head2 decode_record_from_result
369
    my $marc_record = $self->decode_record_from_result(@result);
370
371
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
372
373
=cut
374
375
sub decode_record_from_result {
376
    # Result is passed in as array, will get flattened
377
    # and first element will be $result
378
    my ( $self, $result ) = @_;
379
    if ($result->{marc_format} eq 'base64ISO2709') {
380
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
381
    }
382
    elsif ($result->{marc_format} eq 'MARCXML') {
383
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
384
    }
385
    elsif ($result->{marc_format} eq 'ARRAY') {
386
        return $self->_array_to_marc($result->{marc_data_array});
387
    }
388
    else {
389
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
390
    }
391
}
392
393
=head2 max_result_window
367
=head2 max_result_window
394
368
395
Returns the maximum number of results that can be fetched
369
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+154 lines)
Lines 158-163 use Koha::Virtualshelves; Link Here
158
use Koha::SearchFields;
158
use Koha::SearchFields;
159
159
160
use URI::Escape;
160
use URI::Escape;
161
use Mail::Sendmail;
162
use File::Spec;
163
use File::Path qw(mkpath);
164
use Koha::Email;
165
use Koha::UploadedFiles;
166
use POSIX qw(strftime);
167
use Digest::MD5 qw(md5_hex);
168
use Encode qw(encode);
161
169
162
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
170
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
163
# create a new CGI object
171
# create a new CGI object
Lines 497-502 my $total = 0; # the total results for the whole set Link Here
497
my $facets; # this object stores the faceted results that display on the left-hand of the results page
505
my $facets; # this object stores the faceted results that display on the left-hand of the results page
498
my $results_hashref;
506
my $results_hashref;
499
507
508
my $export = $cgi->param('export');
509
my $preferred_format = $cgi->param('export_format');
510
my $export_user_email = undef;
511
512
if ($template_name eq 'catalogue/results.tt' && $export && $preferred_format && C4::Context->preference('SearchEngine') eq 'Elasticsearch') {
513
514
    my $patron = Koha::Patrons->find( $borrowernumber );
515
516
    if ($patron) {
517
        if ($patron->email) {
518
            $export_user_email = $patron->email;
519
        }
520
        else {
521
            die "Unable to fetch user email";
522
        }
523
    }
524
    else {
525
        die "Unable to fetch user";
526
    }
527
528
    if (!($patron && $patron->has_permission({ tools => 'export_catalog' }))) {
529
        die "Missing permission \"export_catalog\" required for exporting search results";
530
    }
531
532
    my $elasticsearch = $searcher->get_elasticsearch();
533
534
    my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0;
535
    my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
536
    my $error;
537
538
    my $results = eval {
539
        $elasticsearch->search(
540
            index => $searcher->index_name,
541
            scroll => '1m', #TODO: Syspref for scroll time limit?
542
            size => 1000,  #TODO: Syspref for batch size?
543
            body => \%export_query
544
        );
545
    };
546
    if ($@) {
547
        $error = $@;
548
        $searcher->process_error($error);
549
    }
550
551
    my @docs;
552
    for my $doc (@{$results->{hits}->{hits}}) {
553
        push @docs, $doc;
554
    }
555
556
    my $scroll_id = $results->{_scroll_id};
557
558
    while (@{$results->{hits}->{hits}}) {
559
        $results = $elasticsearch->scroll(
560
            scroll => '1m',
561
            scroll_id => $scroll_id
562
        );
563
        for my $doc (@{$results->{hits}->{hits}}) {
564
            push @docs, $doc;
565
        }
566
    }
567
568
    my $koha_email = Koha::Email->new();
569
    my %mail;
570
    my $export_from_address = C4::Context->preference('SearchResultMARCExportFromAddress');
571
572
    if (!$error) {
573
        my $encoded_records = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);
574
575
        my %format_extensions = (
576
            'ISO2709' => '.mrc',
577
            'MARCXML' => '.xml'
578
        );
579
580
        my $upload_dir = Koha::UploadedFile->permanent_directory;
581
        my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";
582
        my %export_links;
583
584
        while (my ($format, $data) = each %{$encoded_records}) {
585
            $data = encode('UTF-8', $data);
586
            my $hash = md5_hex($data);
587
            my $category = "search_marc_export";
588
            my $time = strftime "%Y%m%d_%H%M", localtime time;
589
            my $filename = $category . '_' . $time . $format_extensions{$format};
590
            my $file_dir = File::Spec->catfile($upload_dir, $category);
591
            if ( !-d $file_dir) {
592
                mkpath $file_dir or die "Failed to create $file_dir";
593
            }
594
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
595
596
            my $fh = IO::File->new($filepath, "w");
597
598
            if ($fh) {
599
                $fh->binmode;
600
                print $fh $data;
601
                $fh->close;
602
603
                my $size = -s $filepath;
604
                my $file = Koha::UploadedFile->new({
605
                        hashvalue => $hash,
606
                        filename  => $filename,
607
                        dir       => $category,
608
                        filesize  => $size,
609
                        owner     => $borrowernumber,
610
                        uploadcategorycode => 'search_marc_export',
611
                        public    => 0,
612
                        permanent => 1,
613
                    })->store;
614
                my $id = $file->_result()->get_column('id');
615
                $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";
616
            }
617
            else {
618
                die "Failed to write \"$filepath\"";
619
            }
620
        }
621
622
        if (%export_links) {
623
            my $links_output = '';
624
            while (my ($format, $link) = each %export_links) {
625
                $links_output .= "$format: $link\n";
626
            }
627
628
            my $query_string = $query->{query}->{query_string}->{query};
629
            my $links_count = keys %export_links;
630
            my $message = $links_count > 1 ?
631
                "Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead.\n\n" . $links_output : $links_output;
632
633
            %mail = $koha_email->create_message_headers({
634
                    to => $export_user_email,
635
                    from => $export_from_address,
636
                    subject => "Marc export for query: $query_string",
637
                    message => $message,
638
                });
639
        }
640
    }
641
    else {
642
        %mail = $koha_email->create_message_headers({
643
                to => $export_user_email,
644
                from => $export_from_address,
645
                subject => "Marc export error",
646
                message => "An error occurred during marc export: $error",
647
            });
648
    }
649
    sendmail(%mail) || print "Error: $Mail::Sendmail::error\n";
650
651
    $template->param(export_user_email => $export_user_email);
652
}
653
500
eval {
654
eval {
501
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
655
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
502
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
656
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+16 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });
11
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short') });
12
        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
13
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
14
        say $out "System preferences added";
15
    },
16
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +4 lines)
Lines 189-195 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
192
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
192
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
Lines 199-204 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
199
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
199
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
200
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
200
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
201
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
201
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
202
('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
202
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
203
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
203
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
204
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
204
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
205
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
Lines 594-599 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
594
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
595
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
595
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
596
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
596
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
597
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
598
('SearchResultMARCExportFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short'),
599
('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
597
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
600
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
598
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
601
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
599
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
602
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
Lines 464-472 Administration: Link Here
464
        -
464
        -
465
            - "Elasticsearch MARC format: "
465
            - "Elasticsearch MARC format: "
466
            - pref: ElasticsearchMARCFormat
466
            - pref: ElasticsearchMARCFormat
467
              default: "ISO2709"
467
              default: "base64ISO2709"
468
              choices:
468
              choices:
469
                "ISO2709": "ISO2709 (exchange format)"
469
                "base64ISO2709": "ISO2709 (exchange format)"
470
                "ARRAY": "Searchable array"
470
                "ARRAY": "Searchable array"
471
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
471
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
472
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
472
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+18 lines)
Lines 131-136 Searching: Link Here
131
                  1: use
131
                  1: use
132
                  0: "don't use"
132
                  0: "don't use"
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
134
        -
135
            - pref: EnableSearchResultMARCExport
136
              type: boolean
137
              default: yes
138
              choices:
139
                  yes: Enable
140
                  no: Disable
141
            - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
142
        -
143
            - "Limit exported MARC records from search results to a maximum of"
144
            - pref: SearchResultMARCExportLimit
145
              class: integer
146
            - "records."
147
        -
148
            - "Use the from-address"
149
            - pref: SearchResultMARCExportFromAddress
150
              class: short
151
            - "when mailing marc exports of search results."
134
    Results display:
152
    Results display:
135
        -
153
        -
136
            - pref: numSearchResultsDropdown
154
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+16 lines)
Lines 310-315 Link Here
310
                                </div> <!-- /.btn-group -->
310
                                </div> <!-- /.btn-group -->
311
                            [% END %]
311
                            [% END %]
312
312
313
                            [% IF Koha.Preference('EnableSearchResultMARCExport') && Koha.Preference('SearchEngine') == 'Elasticsearch' && CAN_user_tools_export_catalog %]
314
                                <div class="btn-group">
315
                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
316
                                        Export results<span class="caret"></span>
317
                                    </button>
318
                                    <ul class="dropdown-menu">
319
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
320
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
321
                                   </ul>
322
                                </div> <!-- /.btn-group -->
323
                            [% END %]
324
313
                        </div> <!-- /#selection_ops -->
325
                        </div> <!-- /#selection_ops -->
314
                    </div> <!-- /#searchheader -->
326
                    </div> <!-- /#searchheader -->
315
327
Lines 337-342 Link Here
337
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
349
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
338
                [% END %]
350
                [% END %]
339
351
352
                [% IF ( export_user_email ) %]
353
                    <div class="dialog message">Export in progress, an email will results will be sent to [% export_user_email | html %]</div>
354
                [% END %]
355
340
                <!-- Search Results Table -->
356
                <!-- Search Results Table -->
341
                [% IF ( total ) %]
357
                [% IF ( total ) %]
342
                    [% IF ( scan ) %]
358
                    [% IF ( scan ) %]
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-7 / +58 lines)
Lines 139-148 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
139
139
140
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
140
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
141
141
142
    plan tests => 63;
142
    plan tests => 72;
143
143
144
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
144
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
145
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
145
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
146
146
147
    my @mappings = (
147
    my @mappings = (
148
        {
148
        {
Lines 488-494 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
488
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
488
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
489
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
489
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
490
490
491
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
491
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
492
492
493
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
493
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
494
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
494
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 609-619 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
609
609
610
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
610
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
611
611
612
    $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
612
    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
613
613
614
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
614
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
615
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
615
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
616
616
617
    # Search export functionality
618
    # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
619
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
620
621
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
622
623
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
624
625
        $docs = $see->marc_records_to_documents(\@source_docs);
626
627
        # Emulate Elasticsearch response docs structure
628
        my @es_response_docs = map { { _source => $_ } } @{$docs};
629
630
        my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');
631
632
        # $large_marc_record should not have been encoded as ISO2709
633
        # since exceeds maximum size, see above
634
        my @tmp = ($marc_record_1, $marc_record_2);
635
        is(
636
            $records_data->{ISO2709},
637
            join('', map { $_->as_usmarc() } @tmp),
638
            "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
639
        );
640
641
        my $expected_marc_xml = join("\n",
642
            MARC::File::XML::header(),
643
            MARC::File::XML::record($large_marc_record, 'MARC21'),
644
            MARC::File::XML::footer()
645
        );
646
647
        is(
648
            $records_data->{MARCXML},
649
            $expected_marc_xml,
650
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
651
        );
652
653
        $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');
654
655
        $expected_marc_xml = join("\n",
656
            MARC::File::XML::header(),
657
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
658
            MARC::File::XML::footer()
659
        );
660
661
        is(
662
            $records_data->{MARCXML},
663
            $expected_marc_xml,
664
            "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
665
        );
666
667
    }
668
617
    push @mappings, {
669
    push @mappings, {
618
        name => 'title',
670
        name => 'title',
619
        type => 'string',
671
        type => 'string',
Lines 746-752 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
746
798
747
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
799
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
748
800
749
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
801
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
750
802
751
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
803
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
752
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
804
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 757-763 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
757
    plan tests => 5;
809
    plan tests => 5;
758
810
759
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
811
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
760
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
812
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
761
813
762
    my $builder = t::lib::TestBuilder->new;
814
    my $builder = t::lib::TestBuilder->new;
763
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
815
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
764
- 

Return to bug 27859