View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-33 / +181 lines)
Lines 40-47 use YAML::XS; Link Here
40
40
41
use List::Util qw( sum0 );
41
use List::Util qw( sum0 );
42
use MARC::File::XML;
42
use MARC::File::XML;
43
use MIME::Base64 qw( encode_base64 );
43
use MIME::Base64 qw(encode_base64 decode_base64);
44
use Encode qw( encode );
44
use Encode qw(encode decode);
45
use Business::ISBN;
45
use Business::ISBN;
46
use Scalar::Util qw( looks_like_number );
46
use Scalar::Util qw( looks_like_number );
47
47
Lines 540-546 sub marc_records_to_documents { Link Here
540
    my $control_fields_rules = $rules->{control_fields};
540
    my $control_fields_rules = $rules->{control_fields};
541
    my $data_fields_rules = $rules->{data_fields};
541
    my $data_fields_rules = $rules->{data_fields};
542
    my $marcflavour = lc C4::Context->preference('marcflavour');
542
    my $marcflavour = lc C4::Context->preference('marcflavour');
543
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
544
543
545
    my @record_documents;
544
    my @record_documents;
546
545
Lines 692-729 sub marc_records_to_documents { Link Here
692
                }
691
                }
693
            }
692
            }
694
        }
693
        }
694
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
695
695
696
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
696
        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
697
        $record->encoding('UTF-8');
697
            $record,
698
        if ($use_array) {
698
            $preferred_format,
699
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
699
            $marcflavour
700
            $record_document->{'marc_format'} = 'ARRAY';
700
        );
701
702
        if ($preferred_format eq 'ARRAY') {
703
            $record_document->{'marc_data_array'} = $encoded_record;
701
        } else {
704
        } else {
702
            my @warnings;
705
            $record_document->{'marc_data'} = $encoded_record;
703
            {
706
        }
704
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
707
        $record_document->{'marc_format'} = $format;
705
                local $SIG{__WARN__} = sub {
708
706
                    push @warnings, $_[0];
709
        push @record_documents, $record_document;
707
                };
710
    }
708
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
711
    return \@record_documents;
709
            }
712
}
710
            if (@warnings) {
713
711
                # Suppress warnings if record length exceeded
714
=head2 search_document_marc_record_encode($record, $format, $marcflavour)
712
                unless (substr($record->leader(), 0, 5) eq '99999') {
715
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
713
                    foreach my $warning (@warnings) {
716
714
                        carp $warning;
717
Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
715
                    }
718
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.
719
720
=over 4
721
722
=item C<$record>
723
724
A MARC::Record object
725
726
=item C<$marcflavour>
727
728
The marcflavour to use
729
730
=back
731
732
=cut
733
734
sub search_document_marc_record_encode {
735
    my ($self, $record, $format, $marcflavour) = @_;
736
737
    $record->encoding('UTF-8');
738
739
    if ($format eq 'ARRAY') {
740
        return ($self->_marc_to_array($record), $format);
741
    }
742
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
743
        my @warnings;
744
        my $marc_data;
745
        {
746
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
747
            local $SIG{__WARN__} = sub {
748
                push @warnings, $_[0];
749
            };
750
            $marc_data = $record->as_usmarc();
751
        }
752
        if (@warnings) {
753
            # Suppress warnings if record length exceeded
754
            unless (substr($record->leader(), 0, 5) eq '99999') {
755
                foreach my $warning (@warnings) {
756
                    carp $warning;
716
                }
757
                }
717
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
718
                $record_document->{'marc_format'} = 'MARCXML';
719
            }
758
            }
720
            else {
759
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
721
                $record_document->{'marc_format'} = 'base64ISO2709';
760
        }
761
        else {
762
            if ($format eq 'base64ISO2709') {
763
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
722
            }
764
            }
765
            return ($marc_data, $format);
723
        }
766
        }
724
        push @record_documents, $record_document;
725
    }
767
    }
726
    return \@record_documents;
768
    elsif ($format eq 'MARCXML') {
769
        return (MARC::File::XML::record($record, $marcflavour), $format);
770
    }
771
    else {
772
        # This should be unlikely to happen
773
        croak "Invalid marc record serialization format: $format";
774
    }
775
}
776
777
=head2 search_document_marc_record_decode
778
    my $marc_record = $self->search_document_marc_record_decode(@result);
779
780
Extract marc data from Elasticsearch result and decode to MARC::Record object
781
782
=cut
783
784
sub search_document_marc_record_decode {
785
    # Result is passed in as array, will get flattened
786
    # and first element will be $result
787
    my ($self, $result) = @_;
788
    if ($result->{marc_format} eq 'base64ISO2709') {
789
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
790
    }
791
    elsif ($result->{marc_format} eq 'MARCXML') {
792
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
793
    }
794
    elsif ($result->{marc_format} eq 'ARRAY') {
795
        return $self->_array_to_marc($result->{marc_data_array});
796
    }
797
    else {
798
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
799
    }
800
}
801
802
=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)
803
804
    $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)
805
806
Return marc encoded records from ElasticSearch search result documents. The return value
807
C<$marc_records> is a hashref with encoded records keyed by MARC format.
808
809
=over 4
810
811
=item C<$docs>
812
813
An arrayref of Elasticsearch search documents
814
815
=item C<$preferred_format>
816
817
The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
818
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
819
is set to 'ISO2709'.
820
821
=back
822
823
=cut
824
825
sub search_document_marc_records_encode_from_docs {
826
827
    my ($self, $docs, $preferred_format) = @_;
828
829
    my %encoded_records = (
830
        'ISO2709' => [],
831
        'MARCXML' => []
832
    );
833
834
    unless (exists $encoded_records{$preferred_format}) {
835
       croak "Invalid preferred format: $preferred_format";
836
    }
837
838
    for my $es_record (@{$docs}) {
839
        # Special optimized cases
840
        my $marc_data;
841
        my $resulting_format = $preferred_format;
842
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
843
            $marc_data = $es_record->{_source}{marc_data};
844
        }
845
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
846
            # Stored as UTF-8 encoded binary in index, so needs to be decoded
847
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
848
        }
849
        else {
850
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
851
            my $marcflavour = lc C4::Context->preference('marcflavour');
852
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
853
        }
854
        push @{$encoded_records{$resulting_format}}, $marc_data;
855
    }
856
    if (@{$encoded_records{'ISO2709'}}) {
857
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
858
    }
859
    else {
860
        delete $encoded_records{'ISO2709'};
861
    }
862
863
    if (@{$encoded_records{'MARCXML'}}) {
864
        $encoded_records{'MARCXML'} = join("\n",
865
            MARC::File::XML::header(),
866
            join("\n", @{$encoded_records{'MARCXML'}}),
867
            MARC::File::XML::footer()
868
        );
869
    }
870
    else {
871
        delete $encoded_records{'MARCXML'};
872
    }
873
874
    return \%encoded_records;
727
}
875
}
728
876
729
=head2 _marc_to_array($record)
877
=head2 _marc_to_array($record)
Lines 797-814 sub _array_to_marc { Link Here
797
    $record->leader($data->{leader});
945
    $record->leader($data->{leader});
798
    for my $field (@{$data->{fields}}) {
946
    for my $field (@{$data->{fields}}) {
799
        my $tag = (keys %{$field})[0];
947
        my $tag = (keys %{$field})[0];
800
        $field = $field->{$tag};
948
        my $field_data = $field->{$tag};
801
        my $marc_field;
949
        my $marc_field;
802
        if (ref($field) eq 'HASH') {
950
        if (ref($field_data) eq 'HASH') {
803
            my @subfields;
951
            my @subfields;
804
            foreach my $subfield (@{$field->{subfields}}) {
952
            foreach my $subfield (@{$field_data->{subfields}}) {
805
                my $code = (keys %{$subfield})[0];
953
                my $code = (keys %{$subfield})[0];
806
                push @subfields, $code;
954
                push @subfields, $code;
807
                push @subfields, $subfield->{$code};
955
                push @subfields, $subfield->{$code};
808
            }
956
            }
809
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
957
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
810
        } else {
958
        } else {
811
            $marc_field = MARC::Field->new($tag, $field)
959
            $marc_field = MARC::Field->new($tag, $field_data)
812
        }
960
        }
813
        $record->append_fields($marc_field);
961
        $record->append_fields($marc_field);
814
    }
962
    }
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-29 / +3 lines)
Lines 50-56 use Koha::SearchEngine::Search; Link Here
50
use Koha::Exceptions::Elasticsearch;
50
use Koha::Exceptions::Elasticsearch;
51
use MARC::Record;
51
use MARC::Record;
52
use MARC::File::XML;
52
use MARC::File::XML;
53
use MIME::Base64 qw( decode_base64 );
54
53
55
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
54
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
56
55
Lines 164-170 sub search_compat { Link Here
164
    my $index = $offset;
163
    my $index = $offset;
165
    my $hits = $results->{'hits'};
164
    my $hits = $results->{'hits'};
166
    foreach my $es_record (@{$hits->{'hits'}}) {
165
    foreach my $es_record (@{$hits->{'hits'}}) {
167
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
166
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
168
    }
167
    }
169
168
170
    # consumers of this expect a name-spaced result, we provide the default
169
    # consumers of this expect a name-spaced result, we provide the default
Lines 225-231 sub search_auth_compat { Link Here
225
            # it's not reproduced here yet.
224
            # it's not reproduced here yet.
226
            my $authtype           = $rs->single;
225
            my $authtype           = $rs->single;
227
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
226
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
228
            my $marc               = $self->decode_record_from_result($record);
227
            my $marc               = $self->search_document_marc_record_decode($record);
229
            my $mainentry          = $marc->field($auth_tag_to_report);
228
            my $mainentry          = $marc->field($auth_tag_to_report);
230
            my $reported_tag;
229
            my $reported_tag;
231
            if ($mainentry) {
230
            if ($mainentry) {
Lines 345-351 sub simple_search_compat { Link Here
345
    my @records;
344
    my @records;
346
    my $hits = $results->{'hits'};
345
    my $hits = $results->{'hits'};
347
    foreach my $es_record (@{$hits->{'hits'}}) {
346
    foreach my $es_record (@{$hits->{'hits'}}) {
348
        push @records, $self->decode_record_from_result($es_record->{'_source'});
347
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
349
    }
348
    }
350
    return (undef, \@records, $hits->{'total'});
349
    return (undef, \@records, $hits->{'total'});
351
}
350
}
Lines 365-395 sub extract_biblionumber { Link Here
365
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
364
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
366
}
365
}
367
366
368
=head2 decode_record_from_result
369
    my $marc_record = $self->decode_record_from_result(@result);
370
371
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
372
373
=cut
374
375
sub decode_record_from_result {
376
    # Result is passed in as array, will get flattened
377
    # and first element will be $result
378
    my ( $self, $result ) = @_;
379
    if ($result->{marc_format} eq 'base64ISO2709') {
380
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
381
    }
382
    elsif ($result->{marc_format} eq 'MARCXML') {
383
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
384
    }
385
    elsif ($result->{marc_format} eq 'ARRAY') {
386
        return $self->_array_to_marc($result->{marc_data_array});
387
    }
388
    else {
389
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
390
    }
391
}
392
393
=head2 max_result_window
367
=head2 max_result_window
394
368
395
Returns the maximum number of results that can be fetched
369
Returns the maximum number of results that can be fetched
(-)a/catalogue/search.pl (+153 lines)
Lines 158-163 use Koha::Virtualshelves; Link Here
158
use Koha::SearchFields;
158
use Koha::SearchFields;
159
159
160
use URI::Escape;
160
use URI::Escape;
161
use Mail::Sendmail;
162
use File::Spec;
163
use File::Path qw(mkpath);
164
use Koha::Email;
165
use Koha::UploadedFiles;
166
use POSIX qw(strftime);
167
use Digest::MD5 qw(md5_hex);
168
use Encode qw(encode);
161
169
162
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
170
my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
163
# create a new CGI object
171
# create a new CGI object
Lines 536-541 my $total = 0; # the total results for the whole set Link Here
536
my $facets; # this object stores the faceted results that display on the left-hand of the results page
544
my $facets; # this object stores the faceted results that display on the left-hand of the results page
537
my $results_hashref;
545
my $results_hashref;
538
546
547
my $export = $cgi->param('export');
548
my $preferred_format = $cgi->param('export_format');
549
my $export_user_email = undef;
550
551
if ($template_name eq 'catalogue/results.tt' && $export && $preferred_format && C4::Context->preference('SearchEngine') eq 'Elasticsearch') {
552
553
    my $patron = Koha::Patrons->find( $borrowernumber );
554
555
    if ($patron) {
556
        if ($patron->email) {
557
            $export_user_email = $patron->email;
558
        }
559
        else {
560
            die "Unable to fetch user email";
561
        }
562
    }
563
    else {
564
        die "Unable to fetch user";
565
    }
566
567
    if (!($patron && $patron->has_permission({ tools => 'export_catalog' }))) {
568
        die "Missing permission \"export_catalog\" required for exporting search results";
569
    }
570
571
    my $elasticsearch = $searcher->get_elasticsearch();
572
573
    my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0;
574
    my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
575
    my $error;
576
577
    my $results = eval {
578
        $elasticsearch->search(
579
            index => $searcher->index_name,
580
            scroll => '1m', #TODO: Syspref for scroll time limit?
581
            size => 1000,  #TODO: Syspref for batch size?
582
            body => \%export_query
583
        );
584
    };
585
    if ($@) {
586
        $error = $@;
587
        $searcher->process_error($error);
588
    }
589
590
    my @docs;
591
    for my $doc (@{$results->{hits}->{hits}}) {
592
        push @docs, $doc;
593
    }
594
595
    my $scroll_id = $results->{_scroll_id};
596
597
    while (@{$results->{hits}->{hits}}) {
598
        $results = $elasticsearch->scroll(
599
            scroll => '1m',
600
            scroll_id => $scroll_id
601
        );
602
        for my $doc (@{$results->{hits}->{hits}}) {
603
            push @docs, $doc;
604
        }
605
    }
606
607
    my $koha_email = Koha::Email->new();
608
    my %mail;
609
610
    if (!$error) {
611
        my $encoded_records = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);
612
613
        my %format_extensions = (
614
            'ISO2709' => '.mrc',
615
            'MARCXML' => '.xml'
616
        );
617
618
        my $upload_dir = Koha::UploadedFile->permanent_directory;
619
        my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";
620
        my %export_links;
621
622
        while (my ($format, $data) = each %{$encoded_records}) {
623
            $data = encode('UTF-8', $data);
624
            my $hash = md5_hex($data);
625
            my $category = "search_marc_export";
626
            my $time = strftime "%Y%m%d_%H%M", localtime time;
627
            my $filename = $category . '_' . $time . $format_extensions{$format};
628
            my $file_dir = File::Spec->catfile($upload_dir, $category);
629
            if ( !-d $file_dir) {
630
                mkpath $file_dir or die "Failed to create $file_dir";
631
            }
632
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
633
634
            my $fh = IO::File->new($filepath, "w");
635
636
            if ($fh) {
637
                $fh->binmode;
638
                print $fh $data;
639
                $fh->close;
640
641
                my $size = -s $filepath;
642
                my $file = Koha::UploadedFile->new({
643
                        hashvalue => $hash,
644
                        filename  => $filename,
645
                        dir       => $category,
646
                        filesize  => $size,
647
                        owner     => $borrowernumber,
648
                        uploadcategorycode => 'search_marc_export',
649
                        public    => 0,
650
                        permanent => 1,
651
                    })->store;
652
                my $id = $file->_result()->get_column('id');
653
                $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";
654
            }
655
            else {
656
                die "Failed to write \"$filepath\"";
657
            }
658
        }
659
660
        if (%export_links) {
661
            my $links_output = '';
662
            while (my ($format, $link) = each %export_links) {
663
                $links_output .= "$format: $link\n";
664
            }
665
666
            my $query_string = $query->{query}->{query_string}->{query};
667
            my $links_count = keys %export_links;
668
            my $message = $links_count > 1 ?
669
                "Some records exceeded maximum size supported by ISO2709 and was exported as MARCXML instead.\n\n" . $links_output : $links_output;
670
671
            %mail = $koha_email->create_message_headers({
672
                    to => $export_user_email,
673
                    from => 'noreply@ub.gu.se',
674
                    subject => "Marc export for query: $query_string",
675
                    message => $message,
676
                });
677
        }
678
    }
679
    else {
680
        %mail = $koha_email->create_message_headers({
681
                to => $export_user_email,
682
                from => 'noreply@ub.gu.se',
683
                subject => "Marc export error",
684
                message => "An error occured during marc export: $error",
685
            });
686
    }
687
    sendmail(%mail) || print "Error: $Mail::Sendmail::error\n";
688
689
    $template->param(export_user_email => $export_user_email);
690
}
691
539
eval {
692
eval {
540
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
693
    my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
541
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
694
    ( $error, $results_hashref, $facets ) = $searcher->search_compat(
(-)a/installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl (+9 lines)
Line 0 Link Here
1
$DBversion = 'XXX';  # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
  $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });
4
  $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });
5
  $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
6
  $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
7
8
  #NewVersion( $DBversion, XXXXX, "Add EnableSearchResultMARCExport system prefernce");
9
}
(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +3 lines)
Lines 189-195 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
189
('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
190
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
191
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
192
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
192
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
193
('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
194
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
195
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
Lines 199-204 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
199
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
199
('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
200
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
200
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
201
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
201
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
202
('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
202
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
203
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
203
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
204
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
204
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
205
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
Lines 593-598 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
593
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
594
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
594
('SearchLimitLibrary', 'both', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
595
('SearchLimitLibrary', 'both', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
595
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
596
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
597
('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
596
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
598
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
597
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
599
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
598
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
600
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
Lines 462-470 Administration: Link Here
462
        -
462
        -
463
            - "Elasticsearch MARC format: "
463
            - "Elasticsearch MARC format: "
464
            - pref: ElasticsearchMARCFormat
464
            - pref: ElasticsearchMARCFormat
465
              default: "ISO2709"
465
              default: "base64ISO2709"
466
              choices:
466
              choices:
467
                "ISO2709": "ISO2709 (exchange format)"
467
                "base64ISO2709": "ISO2709 (exchange format)"
468
                "ARRAY": "Searchable array"
468
                "ARRAY": "Searchable array"
469
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
469
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
470
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
470
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+13 lines)
Lines 131-136 Searching: Link Here
131
                  1: use
131
                  1: use
132
                  0: "don't use"
132
                  0: "don't use"
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
133
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
134
        -
135
            - pref: EnableSearchResultMARCExport
136
              type: boolean
137
              default: yes
138
              choices:
139
                  yes: Enable
140
                  no: Disable
141
            - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
142
        -
143
            - "Limit exported MARC records from search results to a maximum of"
144
            - pref: SearchResultMARCExportLimit
145
              class: integer
146
            - "records."
134
    Results display:
147
    Results display:
135
        -
148
        -
136
            - pref: numSearchResultsDropdown
149
            - pref: numSearchResultsDropdown
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+16 lines)
Lines 312-317 Link Here
312
                                </div> <!-- /.btn-group -->
312
                                </div> <!-- /.btn-group -->
313
                            [% END %]
313
                            [% END %]
314
314
315
                            [% IF Koha.Preference('EnableSearchResultMARCExport') && Koha.Preference('SearchEngine') == 'Elasticsearch' && CAN_user_tools_export_catalog %]
316
                                <div class="btn-group">
317
                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
318
                                        Export results<span class="caret"></span>
319
                                    </button>
320
                                    <ul class="dropdown-menu">
321
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
322
                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
323
                                   </ul>
324
                                </div> <!-- /.btn-group -->
325
                            [% END %]
326
315
                        </div> <!-- /#selection_ops -->
327
                        </div> <!-- /#selection_ops -->
316
                    </div> <!-- /#searchheader -->
328
                    </div> <!-- /#searchheader -->
317
329
Lines 339-344 Link Here
339
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
351
                    <div class="dialog alert"><p><strong>Error:</strong> [% query_error | html %]</p></div>
340
                [% END %]
352
                [% END %]
341
353
354
                [% IF ( export_user_email ) %]
355
                    <div class="dialog message">Export in progress, an email will results will be sent to [% export_user_email | html %]</div>
356
                [% END %]
357
342
                <!-- Search Results Table -->
358
                <!-- Search Results Table -->
343
                [% IF ( total ) %]
359
                [% IF ( total ) %]
344
                    [% IF ( scan ) %]
360
                    [% IF ( scan ) %]
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-7 / +58 lines)
Lines 137-146 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
137
137
138
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
138
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
139
139
140
    plan tests => 63;
140
    plan tests => 72;
141
141
142
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
142
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
143
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
143
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
144
144
145
    my @mappings = (
145
    my @mappings = (
146
        {
146
        {
Lines 486-492 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
486
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
486
    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
487
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
487
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
488
488
489
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
489
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
490
490
491
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
491
    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
492
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
492
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 607-617 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
607
607
608
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
608
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
609
609
610
    $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
610
    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
611
611
612
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
612
    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
613
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
613
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
614
614
615
    # Search export functionality
616
    # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
617
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
618
619
    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
620
621
        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
622
623
        $docs = $see->marc_records_to_documents(\@source_docs);
624
625
        # Emulate Elasticsearch response docs structure
626
        my @es_response_docs = map { { _source => $_ } } @{$docs};
627
628
        my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');
629
630
        # $large_marc_record should not have been encoded as ISO2709
631
        # since exceeds maximum size, see above
632
        my @tmp = ($marc_record_1, $marc_record_2);
633
        is(
634
            $records_data->{ISO2709},
635
            join('', map { $_->as_usmarc() } @tmp),
636
            "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
637
        );
638
639
        my $expected_marc_xml = join("\n",
640
            MARC::File::XML::header(),
641
            MARC::File::XML::record($large_marc_record, 'MARC21'),
642
            MARC::File::XML::footer()
643
        );
644
645
        is(
646
            $records_data->{MARCXML},
647
            $expected_marc_xml,
648
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
649
        );
650
651
        $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');
652
653
        $expected_marc_xml = join("\n",
654
            MARC::File::XML::header(),
655
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
656
            MARC::File::XML::footer()
657
        );
658
659
        is(
660
            $records_data->{MARCXML},
661
            $expected_marc_xml,
662
            "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
663
        );
664
665
    }
666
615
    push @mappings, {
667
    push @mappings, {
616
        name => 'title',
668
        name => 'title',
617
        type => 'string',
669
        type => 'string',
Lines 744-750 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
744
796
745
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
797
    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
746
798
747
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
799
    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
748
800
749
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
801
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
750
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
802
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 755-761 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
755
    plan tests => 5;
807
    plan tests => 5;
756
808
757
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
809
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
758
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
810
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
759
811
760
    my $builder = t::lib::TestBuilder->new;
812
    my $builder = t::lib::TestBuilder->new;
761
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
813
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
762
- 

Return to bug 27859