View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-33 / +181 lines)
Lines 41-48 use YAML::XS; Link Here
41
41
42
use List::Util qw( sum0 );
42
use List::Util qw( sum0 );
43
use MARC::File::XML;
43
use MARC::File::XML;
44
use MIME::Base64 qw( encode_base64 );
44
use MIME::Base64 qw(encode_base64 decode_base64);
45
use Encode qw( encode );
45
use Encode qw(encode decode);
46
use Business::ISBN;
46
use Business::ISBN;
47
use Scalar::Util qw( looks_like_number );
47
use Scalar::Util qw( looks_like_number );
48
48
Lines 541-547 sub marc_records_to_documents { Link Here
541
    my $control_fields_rules = $rules->{control_fields};
541
    my $control_fields_rules = $rules->{control_fields};
542
    my $data_fields_rules = $rules->{data_fields};
542
    my $data_fields_rules = $rules->{data_fields};
543
    my $marcflavour = lc C4::Context->preference('marcflavour');
543
    my $marcflavour = lc C4::Context->preference('marcflavour');
544
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
545
544
546
    my @record_documents;
545
    my @record_documents;
547
546
Lines 741-778 sub marc_records_to_documents { Link Here
741
                }
740
                }
742
            }
741
            }
743
        }
742
        }
743
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
744
744
745
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
745
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
746
        $record->encoding('UTF-8');
746
            $record,
747
        if ($use_array) {
747
            $preferred_format,
748
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
748
            $marcflavour
749
            $record_document->{'marc_format'} = 'ARRAY';
749
        );
750
751
        if ($preferred_format eq 'ARRAY') {
752
            $record_document->{'marc_data_array'} = $encoded_record;
750
        } else {
753
        } else {
751
            my @warnings;
754
            $record_document->{'marc_data'} = $encoded_record;
752
            {
755
        }
753
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
756
        $record_document->{'marc_format'} = $format;
754
                local $SIG{__WARN__} = sub {
757
755
                    push @warnings, $_[0];
758
        push @record_documents, $record_document;
756
                };
759
    }
757
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
760
    return \@record_documents;
758
            }
761
}
759
            if (@warnings) {
762
760
                # Suppress warnings if record length exceeded
763
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
761
                unless (substr($record->leader(), 0, 5) eq '99999') {
764
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
762
                    foreach my $warning (@warnings) {
765
763
                        carp $warning;
766
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
764
                    }
767
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.
768
769
=over 4
770
771
=item C<$record>
772
773
A MARC::Record object
774
775
=item C<$marcflavour>
776
777
The marcflavour to use
778
779
=back
780
781
=cut
782
783
sub search_document_marc_record_encode {
784
    my ($self, $record, $format, $marcflavour) = @_;
785
786
    $record->encoding('UTF-8');
787
788
    if ($format eq 'ARRAY') {
789
        return ($self->_marc_to_array($record), $format);
790
    }
791
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
792
        my @warnings;
793
        my $marc_data;
794
        {
795
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
796
            local $SIG{__WARN__} = sub {
797
                push @warnings, $_[0];
798
            };
799
            $marc_data = $record->as_usmarc();
800
        }
801
        if (@warnings) {
802
            # Suppress warnings if record length exceeded
803
            unless (substr($record->leader(), 0, 5) eq '99999') {
804
                foreach my $warning (@warnings) {
805
                    carp $warning;
765
                }
806
                }
766
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
767
                $record_document->{'marc_format'} = 'MARCXML';
768
            }
807
            }
769
            else {
808
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
770
                $record_document->{'marc_format'} = 'base64ISO2709';
809
        }
810
        else {
811
            if ($format eq 'base64ISO2709') {
812
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
771
            }
813
            }
814
            return ($marc_data, $format);
772
        }
815
        }
773
        push @record_documents, $record_document;
774
    }
816
    }
775
    return \@record_documents;
817
    elsif ($format eq 'MARCXML') {
818
        return (MARC::File::XML::record($record, $marcflavour), $format);
819
    }
820
    else {
821
        # This should be unlikely to happen
822
        croak "Invalid marc record serialization format: $format";
823
    }
824
}
825
826
=head2 search_document_marc_record_decode
827
    my $marc_record = $self->search_document_marc_record_decode(@result);
828
829
Extract marc data from Elasticsearch result and decode to MARC::Record object
830
831
=cut
832
833
sub search_document_marc_record_decode {
834
    # Result is passed in as array, will get flattened
835
    # and first element will be $result
836
    my ($self, $result) = @_;
837
    if ($result->{marc_format} eq 'base64ISO2709') {
838
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
839
    }
840
    elsif ($result->{marc_format} eq 'MARCXML') {
841
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
842
    }
843
    elsif ($result->{marc_format} eq 'ARRAY') {
844
        return $self->_array_to_marc($result->{marc_data_array});
845
    }
846
    else {
847
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
848
    }
849
}
850
851
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)
852
853
    $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)
854
855
Return marc encoded records from ElasticSearch search result documents. The return value
856
C<$marc_records> is a hashref with encoded records keyed by MARC format.
857
858
=over 4
859
860
=item C<$docs>
861
862
An arrayref of Elasticsearch search documents
863
864
=item C<$preferred_format>
865
866
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
867
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
868
is set to 'ISO2709'.
869
870
=back
871
872
=cut
873
874
sub search_document_marc_records_encode_from_docs {
875
876
    my ($self, $docs, $preferred_format) = @_;
877
878
    my %encoded_records = (
879
        'ISO2709' => [],
880
        'MARCXML' => []
881
    );
882
883
    unless (exists $encoded_records{$preferred_format}) {
884
       croak "Invalid preferred format: $preferred_format";
885
    }
886
887
    for my $es_record (@{$docs}) {
888
        # Special optimized cases
889
        my $marc_data;
890
        my $resulting_format = $preferred_format;
891
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
892
            $marc_data = $es_record->{_source}{marc_data};
893
        }
894
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
895
            # Stored as UTF-8 encoded binary in index, so needs to be decoded
896
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
897
        }
898
        else {
899
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
900
            my $marcflavour = lc C4::Context->preference('marcflavour');
901
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
902
        }
903
        push @{$encoded_records{$resulting_format}}, $marc_data;
904
    }
905
    if (@{$encoded_records{'ISO2709'}}) {
906
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
907
    }
908
    else {
909
        delete $encoded_records{'ISO2709'};
910
    }
911
912
    if (@{$encoded_records{'MARCXML'}}) {
913
        $encoded_records{'MARCXML'} = join("\n",
914
            MARC::File::XML::header(),
915
            join("\n", @{$encoded_records{'MARCXML'}}),
916
            MARC::File::XML::footer()
917
        );
918
    }
919
    else {
920
        delete $encoded_records{'MARCXML'};
921
    }
922
923
    return \%encoded_records;
776
}
924
}
777
925
778
=head2 _marc_to_array($record)
926
=head2 _marc_to_array($record)
Lines 846-863 sub _array_to_marc { Link Here
846
    $record->leader($data->{leader});
994
    $record->leader($data->{leader});
847
    for my $field (@{$data->{fields}}) {
995
    for my $field (@{$data->{fields}}) {
848
        my $tag = (keys %{$field})[0];
996
        my $tag = (keys %{$field})[0];
849
        $field = $field->{$tag};
997
        my $field_data = $field->{$tag};
850
        my $marc_field;
998
        my $marc_field;
851
        if (ref($field) eq 'HASH') {
999
        if (ref($field_data) eq 'HASH') {
852
            my @subfields;
1000
            my @subfields;
853
            foreach my $subfield (@{$field->{subfields}}) {
1001
            foreach my $subfield (@{$field_data->{subfields}}) {
854
                my $code = (keys %{$subfield})[0];
1002
                my $code = (keys %{$subfield})[0];
855
                push @subfields, $code;
1003
                push @subfields, $code;
856
                push @subfields, $subfield->{$code};
1004
                push @subfields, $subfield->{$code};
857
            }
1005
            }
858
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
1006
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
859
        } else {
1007
        } else {
860
            $marc_field = MARC::Field->new($tag, $field)
1008
            $marc_field = MARC::Field->new($tag, $field_data)
861
        }
1009
        }
862
        $record->append_fields($marc_field);
1010
        $record->append_fields($marc_field);
863
    }
1011
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-29 / +3 lines)
Lines 50-56 use Koha::SearchEngine::Search; Link Here
50
use Koha::Exceptions::Elasticsearch;
50
use Koha::Exceptions::Elasticsearch;
51
use MARC::Record;
51
use MARC::Record;
52
use MARC::File::XML;
52
use MARC::File::XML;
53
use MIME::Base64 qw( decode_base64 );
54
use JSON;
53
use JSON;
55
54
56
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
55
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
Lines 173-179 sub search_compat { Link Here
173
    my $index = $offset;
172
    my $index = $offset;
174
    my $hits = $results->{'hits'};
173
    my $hits = $results->{'hits'};
175
    foreach my $es_record (@{$hits->{'hits'}}) {
174
    foreach my $es_record (@{$hits->{'hits'}}) {
176
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
175
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
177
    }
176
    }
178
177
179
    # consumers of this expect a name-spaced result, we provide the default
178
    # consumers of this expect a name-spaced result, we provide the default
Lines 234-240 sub search_auth_compat { Link Here
234
            # it's not reproduced here yet.
233
            # it's not reproduced here yet.
235
            my $authtype           = $rs->single;
234
            my $authtype           = $rs->single;
236
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
235
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
237
            my $marc               = $self->decode_record_from_result($record);
236
            my $marc               = $self->search_document_marc_record_decode($record);
238
            my $mainentry          = $marc->field($auth_tag_to_report);
237
            my $mainentry          = $marc->field($auth_tag_to_report);
239
            my $reported_tag;
238
            my $reported_tag;
240
            if ($mainentry) {
239
            if ($mainentry) {
Lines 354-360 sub simple_search_compat { Link Here
354
    my @records;
353
    my @records;
355
    my $hits = $results->{'hits'};
354
    my $hits = $results->{'hits'};
356
    foreach my $es_record (@{$hits->{'hits'}}) {
355
    foreach my $es_record (@{$hits->{'hits'}}) {
357
        push @records, $self->decode_record_from_result($es_record->{'_source'});
356
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
358
    }
357
    }
359
    return (undef, \@records, $hits->{'total'});
358
    return (undef, \@records, $hits->{'total'});
360
}
359
}
Lines 374-404 sub extract_biblionumber { Link Here
374
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
373
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
375
}
374
}
376
375
377
=head2 decode_record_from_result
378
    my $marc_record = $self->decode_record_from_result(@result);
379
380
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
381
382
=cut
383
384
sub decode_record_from_result {
385
    # Result is passed in as array, will get flattened
386
    # and first element will be $result
387
    my ( $self, $result ) = @_;
388
    if ($result->{marc_format} eq 'base64ISO2709') {
389
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
390
    }
391
    elsif ($result->{marc_format} eq 'MARCXML') {
392
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
393
    }
394
    elsif ($result->{marc_format} eq 'ARRAY') {
395
        return $self->_array_to_marc($result->{marc_data_array});
396
    }
397
    else {
398
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
399
    }
400
}
401
402
=head2 max_result_window
376
=head2 max_result_window
403
377
404
Returns the maximum number of results that can be fetched
378
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+227 lines)
Lines 158-163 use Koha::Virtualshelves; Link Here
158
use Koha::SearchFields;
158
use Koha::SearchFields;
159
159
160
use URI::Escape;
160
use URI::Escape;
161
use File::Spec;
162
use File::Path qw(mkpath);
163
use Koha::Email;
164
use Koha::UploadedFiles;
165
use POSIX qw(strftime);
166
use Digest::MD5 qw(md5_hex);
167
use Encode qw(encode);
168
use YAML::XS;
169
use Carp qw(croak);
161
170
162
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
171
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
163
# create a new CGI object
172
# create a new CGI object
Lines 501-506 my $total = 0; # the total results for the whole set Link Here
501
my $facets; # this object stores the faceted results that display on the left-hand of the results page
510
my $facets; # this object stores the faceted results that display on the left-hand of the results page
502
my $results_hashref;
511
my $results_hashref;
503
512
513
my $patron = Koha::Patrons->find( $borrowernumber );
514
515
my $export_enabled =
516
    C4::Context->preference('EnableSearchResultMARCExport') &&
517
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
518
    $patron && $patron->has_permission({ tools => 'export_catalog' });
519
520
$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
521
522
if ($export_enabled) {
523
524
    my $export = $cgi->param('export');
525
    my $preferred_format = $cgi->param('export_format');
526
527
    my $export_custom_formats_pref = Load(C4::Context->preference('SearchResultMARCExportCustomFormats'));
528
529
    my $custom_export_formats = {};
530
    if (ref $export_custom_formats_pref eq 'ARRAY') {
531
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
532
            # TODO: Validate on save or throw error here instead of just ignoring?
533
            my $format = $export_custom_formats_pref->[$i];
534
            if (
535
                ref $format->{fields} eq 'ARRAY' &&
536
                @{$format->{fields}} &&
537
                $format->{name}
538
            ) {
539
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
540
                $custom_export_formats->{"custom_$i"} = $format;
541
            }
542
        }
543
    }
544
    $template->param(custom_export_formats => $custom_export_formats);
545
546
    if ($export && $preferred_format) {
547
        my $elasticsearch = $searcher->get_elasticsearch();
548
549
        my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0;
550
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
551
        my $error;
552
553
        my $results = eval {
554
            $elasticsearch->search(
555
                index => $searcher->index_name,
556
                scroll => '1m', #TODO: Syspref for scroll time limit?
557
                size => 1000,  #TODO: Syspref for batch size?
558
                body => \%export_query
559
            );
560
        };
561
        if ($@) {
562
            $error = $@;
563
            $searcher->process_error($error);
564
        }
565
566
        my @docs;
567
        for my $doc (@{$results->{hits}->{hits}}) {
568
            push @docs, $doc;
569
        }
570
571
        my $scroll_id = $results->{_scroll_id};
572
573
        while (@{$results->{hits}->{hits}}) {
574
            $results = $elasticsearch->scroll(
575
                scroll => '1m',
576
                scroll_id => $scroll_id
577
            );
578
            for my $doc (@{$results->{hits}->{hits}}) {
579
                push @docs, $doc;
580
            }
581
        }
582
583
        my $message;
584
        my $export_links_html;
585
586
        if (!$error) {
587
            my $encoded_results = {};
588
589
            if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
590
                $encoded_results = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);
591
            }
592
            elsif(exists $custom_export_formats->{$preferred_format}) {
593
                my $format = $custom_export_formats->{$preferred_format};
594
                my $result;
595
596
                my $doc_get_fields = sub {
597
                    my ($doc, $fields) = @_;
598
                    my @row;
599
                    foreach my $field (@{$fields}) {
600
                        my $values = $doc->{_source}->{$field};
601
                        push @row, $values && @{$values} ? $values : '';
602
                    }
603
                    return \@row;
604
                };
605
606
                my @rows = map { $doc_get_fields->($_, $format->{fields}) } @docs;
607
608
                if($format->{multiple} eq 'ignore') {
609
                    for (my $i = 0; $i < @rows; ++$i) {
610
                        $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
611
                    }
612
                }
613
                elsif($format->{multiple} eq 'newline') {
614
                    if (@{$format->{fields}} == 1) {
615
                        @rows = map { [join("\n", @{$_->[0]})] } @rows;
616
                    }
617
                    else {
618
                        croak "'newline' is only valid for single field export formats";
619
                    }
620
                }
621
                elsif($format->{multiple} eq 'join') {
622
                    for (my $i = 0; $i < @rows; ++$i) {
623
                        $rows[$i] = [map { join("\t", @{$_}) } @{$rows[$i]}];
624
                    }
625
                }
626
                else {
627
                    croak "Invalid 'multiple' option: " . $format->{multiple};
628
                }
629
630
                if (@{$format->{fields}} == 1) {
631
                    @rows = map { $_->[0] } @rows;
632
                }
633
                else {
634
                    # Encode CSV
635
                    for (my $i = 0; $i < @rows; ++$i) {
636
                        $rows[$i] = join(',', map { $_ =~ s/"/""/; "\"$_\"" } @{$rows[$i]});
637
                    }
638
                }
639
                $encoded_results->{$format->{name}} = join("\n", @rows);
640
            }
641
            else {
642
                croak "Invalid export format: $preferred_format";
643
            }
644
645
            my %format_extensions = (
646
                'ISO2709' => '.mrc',
647
                'MARCXML' => '.xml',
648
            );
649
650
            my $upload_dir = Koha::UploadedFile->permanent_directory;
651
            my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";
652
            my %export_links;
653
654
            while (my ($format, $data) = each %{$encoded_results}) {
655
                $data = encode('UTF-8', $data);
656
                my $hash = md5_hex($data);
657
                my $category = "search_marc_export";
658
                my $time = strftime "%Y%m%d_%H%M", localtime time;
659
                my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
660
                my $filename = $category . '_' . $time . $ext;
661
                my $file_dir = File::Spec->catfile($upload_dir, $category);
662
                if ( !-d $file_dir) {
663
                    mkpath $file_dir or die "Failed to create $file_dir";
664
                }
665
                my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
666
667
                my $fh = IO::File->new($filepath, "w");
668
669
                if ($fh) {
670
                    $fh->binmode;
671
                    print $fh $data;
672
                    $fh->close;
673
674
                    my $size = -s $filepath;
675
                    my $file = Koha::UploadedFile->new({
676
                            hashvalue => $hash,
677
                            filename  => $filename,
678
                            dir       => $category,
679
                            filesize  => $size,
680
                            owner     => $borrowernumber,
681
                            uploadcategorycode => 'search_marc_export',
682
                            public    => 0,
683
                            permanent => 1,
684
                        })->store;
685
                    my $id = $file->_result()->get_column('id');
686
                    $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";
687
                }
688
                else {
689
                    croak "Failed to write \"$filepath\"";
690
                }
691
            }
692
693
            while (my ($format, $link) = each %export_links) {
694
                $export_links_html .= "$format: <a href=\"$link\">$link</a>\n";
695
            }
696
            my $query_string = $query->{query}->{query_string}->{query};
697
            my $links_count = keys %export_links;
698
699
            $export_links_html = $links_count > 1 ?
700
            "<p>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead.</p>" . $export_links_html : $export_links_html;
701
702
            my $send_email = C4::Context->preference('SearchResultMARCExportEmail');
703
            if ($send_email) {
704
                if ($patron->email) {
705
                    my $export_from_address = C4::Context->preference('SearchResultMARCExportEmailFromAddress');
706
                    my $export_user_email = $patron->email;
707
                    my $mail = Koha::Email->create({
708
                            to => $export_user_email,
709
                            from => $export_from_address,
710
                            subject => "Marc export for query: $query_string",
711
                            html_body => $export_links_html,
712
                        });
713
                    $mail->send_or_die({ transport => $patron->library->smtp_server->transport });
714
                    $export_links_html .= "<p>An email has been sent to: $export_user_email</p>";
715
716
                }
717
                else {
718
                    $export_links_html .= "<p>Unable to send mail, the current user has no email address set</p>";
719
                }
720
            }
721
            $message = "<p>The export finished successfully:</p>" . $export_links_html;
722
            $template->param(export_message => $message);
723
        }
724
        else {
725
            $message = "An error occurred during marc export: $error",
726
            $template->param(export_error => $message);
727
        }
728
    }
729
}
730
504
eval {
731
eval {
505
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
732
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
506
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
733
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+18 lines)
Line 0 Link Here
1
use Modern::Perl;
2
3
return {
4
    bug_number => "27859",
5
    description => "Add system preferences",
6
    up => sub {
7
        my ($args) = @_;
8
        my ($dbh, $out) = @$args{qw(dbh out)};
9
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });
10
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea') });
11
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo') });
12
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short') });
13
        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });
14
        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
15
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
16
        say $out "System preferences added";
17
    },
18
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +6 lines)
Lines 189-195 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
192
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
192
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
Lines 200-205 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
200
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
200
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
201
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
201
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
202
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
202
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
203
('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
203
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
204
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
204
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
205
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
205
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
206
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
Lines 594-599 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
594
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
595
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
595
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
596
('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
596
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
597
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
598
('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea'),
599
('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo'),
600
('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short'),
601
('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
597
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
602
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
598
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
603
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
599
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
604
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
Lines 472-480 Administration: Link Here
472
        -
472
        -
473
            - "Elasticsearch MARC format: "
473
            - "Elasticsearch MARC format: "
474
            - pref: ElasticsearchMARCFormat
474
            - pref: ElasticsearchMARCFormat
475
              default: "ISO2709"
475
              default: "base64ISO2709"
476
              choices:
476
              choices:
477
                "ISO2709": "ISO2709 (exchange format)"
477
                "base64ISO2709": "ISO2709 (exchange format)"
478
                "ARRAY": "Searchable array"
478
                "ARRAY": "Searchable array"
479
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
479
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
480
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
480
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+47 lines)
Lines 131-136 Searching: Link Here
131
                  1: use
131
                  1: use
132
                  0: "don't use"
132
                  0: "don't use"
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
134
        -
135
            - pref: EnableSearchResultMARCExport
136
              type: boolean
137
              default: yes
138
              choices:
139
                  yes: Enable
140
                  no: Disable
141
            - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
142
        -
143
            - pref: SearchResultMARCExportCustomFormats
144
              type: textarea
145
              syntax: text/x-yaml
146
              class: code
147
            - "Define custom export formats as a YAML list of associative arrays."
148
            - "A format have the required properties \"<strong>name</strong>\", \"<strong>fields</strong>\" and an optional \"<strong>multiple</strong>\".<br />"
149
            - "<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>"
150
            - "<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.</p>"
151
            - "If <strong>fields</strong> contain a only single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.<br />"
152
            - "<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i><br />The behavior when handling fields with multiple values.</p>"
153
            - "<p><i>ignore</i> is the default option, only the first value will be included, the rest ignored.</p>"
154
            - "<p><i>join</i>, multiple values will be contatenated with tab as a separator.</p>"
155
            - "<p><i>newline</i>, a newline will be inserted for each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>"
156
            - "Example:</br>"
157
            - "- name: Biblionumbers<br />"
158
            - "&nbsp;&nbsp;fields: [local-number]<br />"
159
            - "&nbsp;&nbsp;multiple: ignore<br />"
160
            - "- name: Title and author<br />"
161
            - "&nbsp;&nbsp;fields: [title, author]<br />"
162
            - "&nbsp;&nbsp;multiple: join<br />"
163
        -
164
            - "Limit exported MARC records from search results to a maximum of"
165
            - pref: SearchResultMARCExportLimit
166
              class: integer
167
            - "records."
168
        -
169
            - "Send an email with the export results to the current user"
170
            - pref: SearchResultMARCExportEmail
171
              type: boolean
172
              default: no
173
              choices:
174
                  1: Yes
175
                  0: No
176
        -
177
            - "Use the from-address"
178
            - pref: SearchResultMARCExportFromAddress
179
              class: short
180
            - "when mailing search results exports."
134
    Results display:
181
    Results display:
135
        -
182
        -
136
            - pref: numSearchResultsDropdown
183
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+22 lines)
Lines 311-316 Link Here
311
                                </div> <!-- /.btn-group -->
311
                                </div> <!-- /.btn-group -->
312
                            [% END %]
312
                            [% END %]
313
313
314
                            [% IF export_enabled %]
315
                                <div class="btn-group">
316
                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
317
                                        Export all results<span class="caret"></span>
318
                                    </button>
319
                                    <ul class="dropdown-menu">
320
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
321
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
322
                                        [% FOREACH id IN custom_export_formats.keys %]
323
                                            <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
324
                                        [% END %]
325
                                   </ul>
326
                                </div> <!-- /.btn-group -->
327
                            [% END %]
328
314
                        </div> <!-- /#selection_ops -->
329
                        </div> <!-- /#selection_ops -->
315
                    </div> <!-- /#searchheader -->
330
                    </div> <!-- /#searchheader -->
316
331
Lines 339-344 Link Here
339
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
354
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
340
                [% END %]
355
                [% END %]
341
356
357
                [% IF ( export_message ) %]
358
                    <div class="dialog message">[% export_message %]</div>
359
                [% END %]
360
                [% IF ( export_error ) %]
361
                    <div class="dialog error">[% export_error %]</div>
362
                [% END %]
363
342
                <!-- Search Results Table -->
364
                <!-- Search Results Table -->
343
                [% IF ( total ) %]
365
                [% IF ( total ) %]
344
                    [% IF ( scan ) %]
366
                    [% IF ( scan ) %]
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-7 / +58 lines)
Lines 140-149 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
140
140
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
142
142
143
    plan tests => 63;
143
    plan tests => 72;
144
144
145
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
145
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
146
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
146
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
147
147
148
    my @mappings = (
148
    my @mappings = (
149
        {
149
        {
Lines 489-495 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
489
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
489
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
490
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
490
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
491
491
492
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
492
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
493
493
494
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
494
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
495
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
495
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 610-620 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
610
610
611
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
611
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
612
612
613
    $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
613
    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
614
614
615
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
615
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
616
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
616
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
617
617
618
    # Search export functionality
619
    # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
620
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
621
622
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
623
624
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
625
626
        $docs = $see->marc_records_to_documents(\@source_docs);
627
628
        # Emulate Elasticsearch response docs structure
629
        my @es_response_docs = map { { _source => $_ } } @{$docs};
630
631
        my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');
632
633
        # $large_marc_record should not have been encoded as ISO2709
634
        # since exceeds maximum size, see above
635
        my @tmp = ($marc_record_1, $marc_record_2);
636
        is(
637
            $records_data->{ISO2709},
638
            join('', map { $_->as_usmarc() } @tmp),
639
            "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
640
        );
641
642
        my $expected_marc_xml = join("\n",
643
            MARC::File::XML::header(),
644
            MARC::File::XML::record($large_marc_record, 'MARC21'),
645
            MARC::File::XML::footer()
646
        );
647
648
        is(
649
            $records_data->{MARCXML},
650
            $expected_marc_xml,
651
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
652
        );
653
654
        $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');
655
656
        $expected_marc_xml = join("\n",
657
            MARC::File::XML::header(),
658
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
659
            MARC::File::XML::footer()
660
        );
661
662
        is(
663
            $records_data->{MARCXML},
664
            $expected_marc_xml,
665
            "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
666
        );
667
668
    }
669
618
    push @mappings, {
670
    push @mappings, {
619
        name => 'title',
671
        name => 'title',
620
        type => 'string',
672
        type => 'string',
Lines 747-753 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
747
799
748
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
800
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
749
801
750
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
802
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
751
803
752
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
804
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
753
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
805
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 758-764 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
758
    plan tests => 5;
810
    plan tests => 5;
759
811
760
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
812
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
761
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
813
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
762
814
763
    my $builder = t::lib::TestBuilder->new;
815
    my $builder = t::lib::TestBuilder->new;
764
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
816
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
765
- 

Return to bug 27859