Attachment #147936 for bug #27859

View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

Lines 426-431 sub core_types_to_classes { Link Here

(-)a/Koha/BackgroundJob.pm (+1 lines)
426	stage_marc_for_import => 'Koha::BackgroundJob::StageMARCForImport',	426	stage_marc_for_import => 'Koha::BackgroundJob::StageMARCForImport',
427	marc_import_commit_batch => 'Koha::BackgroundJob::MARCImportCommitBatch',	427	marc_import_commit_batch => 'Koha::BackgroundJob::MARCImportCommitBatch',
428	marc_import_revert_batch => 'Koha::BackgroundJob::MARCImportRevertBatch',	428	marc_import_revert_batch => 'Koha::BackgroundJob::MARCImportRevertBatch',
		429	search_result_export => 'Koha::BackgroundJob::SearchResultExport',
429	};	430	};
430	}	431	}
431		432




package Koha::BackgroundJob::SearchResultExport;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Try::Tiny;
use Koha::SearchEngine::Search;

use File::Spec;
use File::Path qw(mkpath);
use Koha::Email;
use Koha::UploadedFiles;
use POSIX qw(strftime);
use Digest::MD5 qw(md5_hex);
use Carp qw(croak);

use base 'Koha::BackgroundJob';

=head1 NAME

Koha::BackgroundJob::SearchResultExport - Export data from search result

This is a subclass of Koha::BackgroundJob.

=head1 API

=head2 Class methods

=head3 job_type

Define the job type of this job: stage_marc_for_import

=cut

sub job_type {
    return 'search_result_export';
}

=head3 process

Perform the export of search records.

=cut

sub process {
    my ( $self, $args ) = @_;

    $self->start;

    my $data = $self->decoded_data;
    my $borrowernumber = $data->{borrowernumber};
    my $elasticsearch_query = $args->{elasticsearch_query};
    my $preferred_format = $args->{preferred_format};
    my $searcher = Koha::SearchEngine::Search->new({
        index => $Koha::SearchEngine::BIBLIOS_INDEX
    });
    my $elasticsearch = $searcher->get_elasticsearch();

    my $results = eval {
        $elasticsearch->search(
            index => $searcher->index_name,
            scroll => '1m', #TODO: Syspref for scroll time limit?
            size => 1000,  #TODO: Syspref for batch size?
            body => $elasticsearch_query
        );
    };
    my @errors;
    push @errors, $@ if $@;

    my @docs;
    my $encoded_results;
    my %export_links;
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};

    if (!@errors) {
        my $scroll_id = $results->{_scroll_id};
        while (@{$results->{hits}->{hits}}) {
            push @docs, @{$results->{hits}->{hits}};
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
            $results = $elasticsearch->scroll(
                scroll => '1m',
                scroll_id => $scroll_id
            );
        }

        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
        }
        else {
            $encoded_results->{$preferred_format->{name}} =
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
        }

        my %format_extensions = (
            'ISO2709' => '.mrc',
            'MARCXML' => '.xml',
        );

        my $upload_dir = Koha::UploadedFile->permanent_directory;

        while (my ($format, $data) = each %{$encoded_results}) {
            my $hash = md5_hex($data);
            my $category = "search_marc_export";
            my $time = strftime "%Y%m%d_%H%M", localtime time;
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
            my $filename = $category . '_' . $time . $ext;
            my $file_dir = File::Spec->catfile($upload_dir, $category);
            if ( !-d $file_dir) {
                unless(mkpath $file_dir) {
                    push @errors, "Failed to create $file_dir";
                    next;
                }
            }
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");

            my $fh = IO::File->new($filepath, "w");

            if ($fh) {
                $fh->binmode;
                print $fh $data;
                $fh->close;

                my $size = -s $filepath;
                my $file = Koha::UploadedFile->new({
                        hashvalue => $hash,
                        filename  => $filename,
                        dir       => $category,
                        filesize  => $size,
                        owner     => $borrowernumber,
                        uploadcategorycode => 'search_marc_export',
                        public    => 0,
                        permanent => 1,
                    })->store;
                my $id = $file->_result()->get_column('id');
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
            }
            else {
                push @errors, "Failed to write \"$filepath\"";
            }
        }
    }
    my $report = {
        export_links => \%export_links,
        total => scalar @docs,
        errors => \@errors,
        query_string => $query_string,
    };
    $data->{report}   = $report;
    if (@errors) {
        $self->set({ progress => 0, status => 'failed' })->store;
    }
    else {
        $self->finish($data);
    }
}

=head3 enqueue

Enqueue the new job

=cut

sub enqueue {
    my ( $self, $args) = @_;
    $self->SUPER::enqueue({
        job_size => $args->{size},
        job_args => $args,
        job_queue => 'long_tasks',
    });
}

1;





use List::Util qw( sum0 );
use MARC::File::XML;
use MIME::Base64 qw(encode_base64 decode_base64);
use Encode qw(encode decode);
use Business::ISBN;
use Scalar::Util qw( looks_like_number );


    my $control_fields_rules = $rules->{control_fields};
    my $data_fields_rules = $rules->{data_fields};
    my $marcflavour = lc C4::Context->preference('marcflavour');
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';

    my @record_documents;


                }
            }
        }
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');

        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
            $record,
            $preferred_format,
            $marcflavour
        );

        if ($preferred_format eq 'ARRAY') {
            $record_document->{'marc_data_array'} = $encoded_record;
        } else {
            $record_document->{'marc_data'} = $encoded_record;
            {
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
                local $SIG{__WARN__} = sub {
                    push @warnings, $_[0];
                };
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
            }
            if (@warnings) {
                # Suppress warnings if record length exceeded
                unless (substr($record->leader(), 0, 5) eq '99999') {
                    foreach my $warning (@warnings) {
                        carp $warning;
                    }
                }
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
                $record_document->{'marc_format'} = 'MARCXML';
            }
            else {
                $record_document->{'marc_format'} = 'base64ISO2709';
            }
        }
        $record_document->{'marc_format'} = $format;


        # Check if there is at least one available item
        if ($self->index eq $BIBLIOS_INDEX) {

                    onloan       => undef,
                    itemlost     => 0,
                })->count;

                $record_document->{available} = $avail_items ? \1 : \0;
            }
        }

    return \@record_documents;
}

=head2 search_document_marc_record_encode($record, $format, $marcflavour)
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)

Encode a MARC::Record to the preferred marc document record format. If record
exceeds ISO2709 maximum size record size and C<$format> is set to
'base64ISO2709' format will fallback to 'MARCXML' instead.

=over 4

=item C<$record>

A MARC::Record object

=item C<$marcflavour>

The marcflavour to use

=back

=cut

sub search_document_marc_record_encode {
    my ($self, $record, $format, $marcflavour) = @_;

    $record->encoding('UTF-8');

    if ($format eq 'ARRAY') {
        return ($self->_marc_to_array($record), $format);
    }
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
        my @warnings;
        my $marc_data;
        {
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
            local $SIG{__WARN__} = sub {
                push @warnings, $_[0];
            };
            $marc_data = $record->as_usmarc();
        }
        if (@warnings) {
            # Suppress warnings if record length exceeded
            unless (substr($record->leader(), 0, 5) eq '99999') {
                foreach my $warning (@warnings) {
                    carp $warning;
                }
            }
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
        }
        else {
            if ($format eq 'base64ISO2709') {
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
            }
            return ($marc_data, $format);
        }
    }
    elsif ($format eq 'MARCXML') {
        return (MARC::File::XML::record($record, $marcflavour), $format);
    }
    else {
        # This should be unlikely to happen
        croak "Invalid marc record serialization format: $format";
    }
}

=head2 search_document_marc_record_decode
    my $marc_record = $self->search_document_marc_record_decode(@result);

Extract marc data from Elasticsearch result and decode to MARC::Record object

=cut

sub search_document_marc_record_decode {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ($self, $result) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 search_documents_encode($docs, $preferred_format)

    $records_data = $self->search_documents_encode($docs, $preferred_format)

Return marc encoded records from ElasticSearch search result documents. The return value
C<$marc_records> is a hashref with encoded records keyed by MARC format.

=over 4

=item C<$docs>

An arrayref of Elasticsearch search documents

=item C<$preferred_format>

The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
is set to 'ISO2709'.

=back

=cut

sub search_documents_encode {

    my ($self, $docs, $preferred_format) = @_;

    my %encoded_records = (
        'ISO2709' => [],
        'MARCXML' => []
    );

    unless (exists $encoded_records{$preferred_format}) {
       croak "Invalid preferred format: $preferred_format";
    }

    for my $es_record (@{$docs}) {
        # Special optimized cases
        my $marc_data;
        my $resulting_format = $preferred_format;
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
            $marc_data = $es_record->{_source}{marc_data};
        }
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
            $marc_data = decode_base64($es_record->{_source}->{marc_data});
        }
        else {
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
            my $marcflavour = lc C4::Context->preference('marcflavour');
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
        }
        push @{$encoded_records{$resulting_format}}, $marc_data;
    }
    if (@{$encoded_records{'ISO2709'}}) {
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
    }
    else {
        delete $encoded_records{'ISO2709'};
    }

    if (@{$encoded_records{'MARCXML'}}) {
        $encoded_records{'MARCXML'} = encode(
            'UTF-8',
            join(
                "\n",
                MARC::File::XML::header(),
                join("\n", @{$encoded_records{'MARCXML'}}),
                MARC::File::XML::footer()
            )
        );
    }
    else {
        delete $encoded_records{'MARCXML'};
    }

    return \%encoded_records;
}

=head2 search_result_export_custom_formats()

    $custom_formats = $self->search_result_export_custom_formats()

Return user defined custom search result export formats.

=cut

sub search_result_export_custom_formats {
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
    my $custom_export_formats = {};

    if (ref $export_custom_formats_pref eq 'ARRAY') {
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
            # TODO: Perhaps validate on save or trow error here instead of just
            # ignoring invalid formats
            my $format = $export_custom_formats_pref->[$i];
            if (
                ref $format->{fields} eq 'ARRAY' &&
                @{$format->{fields}} &&
                $format->{name}
            ) {
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
                $custom_export_formats->{"custom_$i"} = $format;
            }
        }
    }
    return $custom_export_formats;
}

=head2 search_documents_custom_format_encode($docs, $custom_format)

    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)

Return encoded records from ElasticSearch search result documents using a
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
Returns the encoded records.

=over 4

=item C<$docs>

An arrayref of Elasticsearch search documents

=item C<$format>

A hashref with the custom format definition.

=back

=cut

sub search_documents_custom_format_encode {
    my ($self, $docs, $format) = @_;

    my $result;

    my $doc_get_fields = sub {
        my ($doc, $fields) = @_;
        my @row;
        foreach my $field (@{$fields}) {
            my $values = $doc->{_source}->{$field};
            push @row, ref $values eq 'ARRAY' ? $values : [''];
        }
        return \@row;
    };

    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};

    if($format->{multiple} eq 'ignore') {
        for (my $i = 0; $i < @rows; ++$i) {
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
        }
    }
    elsif($format->{multiple} eq 'newline') {
        if (@{$format->{fields}} == 1) {
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
        }
        else {
            croak "'newline' is only valid for single field export formats";
        }
    }
    elsif($format->{multiple} eq 'join') {
        for (my $i = 0; $i < @rows; ++$i) {
            # Escape separator
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
                }
            }
            # Separate multiple values with "|"
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
        }
    }
    else {
        croak "Invalid 'multiple' option: " . $format->{multiple};
    }
    if (@{$format->{fields}} == 1) {
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
    }
    else {
        # Encode CSV
        for (my $i = 0; $i < @rows; ++$i) {
            # Escape quotes
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
                $rows[$i][$j] =~ s/"/""/g;
            }
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
        }
    }

    return encode('UTF-8', join("\n", @rows));
}

=head2 _marc_to_array($record)

    my @fields = _marc_to_array($record)

    $record->leader($data->{leader});
    for my $field (@{$data->{fields}}) {
        my $tag = (keys %{$field})[0];
        my $field_data = $field->{$tag};
        my $marc_field;
        if (ref($field_data) eq 'HASH') {
            my @subfields;
            foreach my $subfield (@{$field_data->{subfields}}) {
                my $code = (keys %{$subfield})[0];
                push @subfields, $code;
                push @subfields, $subfield->{$code};
            }
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
        } else {
            $marc_field = MARC::Field->new($tag, $field_data)
        }
        $record->append_fields($marc_field);
    }




    my $index = $offset;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
    }

    # consumers of this expect a name-spaced result, we provide the default

            # it's not reproduced here yet.
            my $authtype           = $rs->single;
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
            my $marc               = $self->search_document_marc_record_decode($record);
            my $mainentry          = $marc->field($auth_tag_to_report);
            my $reported_tag;
            if ($mainentry) {

    my @records;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
    }
    return (undef, \@records, $hits->{'total'});
}

    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
}

=head2 decode_record_from_result
    my $marc_record = $self->decode_record_from_result(@result);

Extracts marc data from Elasticsearch result and decodes to MARC::Record object

=cut

sub decode_record_from_result {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ( $self, $result ) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 max_result_window

Returns the maximum number of results that can be fetched




use URI::Escape;
use POSIX qw(ceil floor);
use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
use Koha::BackgroundJob::SearchResultExport;

use Koha::ItemTypes;
use Koha::Library::Groups;


use URI::Escape;
use JSON qw( decode_json encode_json );
use Carp qw(croak);

my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
# create a new CGI object

} #/end of the for loop
#$template->param(FEDERATED_RESULTS => \@results_array);

my $patron = Koha::Patrons->find( $borrowernumber );
my $export_enabled =
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
    $patron && $patron->has_permission({ tools => 'export_catalog' });

$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';

if ($export_enabled) {


    my $export = $cgi->param('export');
    my $preferred_format = $cgi->param('export_format');
    my $custom_export_formats = $searcher->search_result_export_custom_formats;

    $template->param(custom_export_formats => $custom_export_formats);

    # TODO: Need to handle $hits = 0?
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;

    if ($export && $preferred_format && $hits) {
        unless (
            $preferred_format eq 'ISO2709' ||
            $preferred_format eq 'MARCXML'
        ) {
            if (!exists $custom_export_formats->{$preferred_format}) {
                croak "Invalid export format: $preferred_format";
            }
            else {
                $preferred_format = $custom_export_formats->{$preferred_format};
            }
        }
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
            size => $size,
            preferred_format => $preferred_format,
            elasticsearch_query => \%export_query
        });
        $template->param(export_job_id => $export_job_id);
    }
}

my $gotonumber = $cgi->param('gotoNumber');
if ( $gotonumber && ( $gotonumber eq 'last' || $gotonumber eq 'first' ) ) {
    $template->{'VARS'}->{'gotoNumber'} = $gotonumber;




use Modern::Perl;

return {
    bug_number => "27859",
    description => "Add system preferences",
    up => sub {
        my ($args) = @_;
        my ($dbh, $out) = @$args{qw(dbh out)};

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";

        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
        say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
    },
}




[% USE Koha %]

[% BLOCK report %]
    [% SET report = job.report %]
    [% IF report %]
        [% IF job.status == 'finished' %]
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
            <ul>
              <li>[% report.total | html %] records exported</li>
              [% IF report.export_links.keys.size > 1 %]
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
              [% END %]
              [% FOREACH format IN report.export_links.keys.sort %]
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
              [% END %]
            </ul>
        [% ELSE %]
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
            <ul>
            [% FOREACH error in report.errors %]
              <li>[% error | html %]</li>
            [% END %]
            </ul>
        [% END %]
    [% END %]
[% END %]

[% BLOCK detail %]
[% END %]

[% BLOCK js %]
[% END %]

Lines 233-238 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
233	'_id': 'marc_import_revert_batch',	233	'_id': 'marc_import_revert_batch',
234	'_str': _("Revert import MARC records")	234	'_str': _("Revert import MARC records")
235	},	235	},
		236	{
		237	'_id': 'search_result_export',
		238	'_str': _("Search result export")
		239	},
236	];	240	];
237		241
238	function get_job_type (job_type) {	242	function get_job_type (job_type) {

Lines 474-482 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
474	-	474	-
475	- "Elasticsearch MARC format: "	475	- "Elasticsearch MARC format: "
476	- pref: ElasticsearchMARCFormat	476	- pref: ElasticsearchMARCFormat
477	default: "ISO2709"	477	default: "base64ISO2709"
478	choices:	478	choices:
479	"ISO2709": "ISO2709 (exchange format)"	479	"base64ISO2709": "ISO2709 (exchange format)"
480	"ARRAY": "Searchable array"	480	"ARRAY": "Searchable array"
481	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.	481	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
482	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.	482	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.




                  1: use
                  0: "don't use"
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
        -
            - pref: EnableElasticsearchSearchResultExport
              type: boolean
              default: yes
              choices:
                  1: Enable
                  0: Disable
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
        -
            - pref: ElasticsearchSearchResultExportCustomFormats
              type: textarea
              syntax: text/x-yaml
              class: code
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
            - <p>The behavior when handling fields with multiple values.</p>
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
            - 'Example:</br>'
            - '- name: Biblionumbers<br />'
            - '&nbsp;&nbsp;fields: [local-number]<br />'
            - '&nbsp;&nbsp;multiple: ignore<br />'
            - '- name: Title and author<br />'
            - '&nbsp;&nbsp;fields: [title, author]<br />'
            - '&nbsp;&nbsp;multiple: join<br /><br />'
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
        -
            - Limit export from search results to a maximum of
            - pref: ElasticsearchSearchResultExportLimit
              class: integer
            - search result items (Elasticsearch only).<br /><br />
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
    Results display:
        -
            - pref: numSearchResultsDropdown

Lines 329-334 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+51 lines)
329	</div> <!-- /.btn-group -->	329	</div> <!-- /.btn-group -->
330	[% END %]	330	[% END %]
331		331
		332	[% IF export_enabled %]
		333	<div class="btn-group">
		334	<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
		335	Export all results<span class="caret"></span>
		336	</button>
		337	<ul class="dropdown-menu">
		338	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
		339	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
		340	[% FOREACH id IN custom_export_formats.keys %]
		341	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=[% id \| uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name \| html %]</a></li>
		342	[% END %]
		343	</ul>
		344	</div> <!-- /.btn-group -->
		345	[% END %]
		346
332	</div> <!-- /#selection_ops -->	347	</div> <!-- /#selection_ops -->
333	</div> <!-- /#searchheader -->	348	</div> <!-- /#searchheader -->
334		349
Lines 362-367 Link Here
362	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>	377	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>
363	[% END %]	378	[% END %]
364		379
		380	[% IF export_job_id %]
		381	<div class="dialog message">
		382	<p>Exporting records, the export will be processed as soon as possible.</p>
		383	[% INCLUDE "job_progress.inc" job_id=export_job_id %]
		384	<p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id \| uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
		385	<div id="job_callback"></div>
		386	</div>
		387	[% END %]
		388
365	<!-- Search Results Table -->	389	<!-- Search Results Table -->
366	[% IF ( total ) %]	390	[% IF ( total ) %]
367	[% IF ( scan ) %]	391	[% IF ( scan ) %]
Lines 781-787 Link Here
781	[% Asset.css("css/humanmsg.css") \| $raw %]	805	[% Asset.css("css/humanmsg.css") \| $raw %]
782	[% Asset.js("lib/jquery/plugins/humanmsg.js") \| $raw %]	806	[% Asset.js("lib/jquery/plugins/humanmsg.js") \| $raw %]
783	[% INCLUDE 'select2.inc' %]	807	[% INCLUDE 'select2.inc' %]
		808	[% INCLUDE 'str/job_progess.inc' %]
		809	[% Asset.js("js/job_progess.js") \| $raw %]
784	<script>	810	<script>
		811	[% IF export_job_id %]
		812	updateProgress([% export_job_id \| html %], function() {
		813	$.getJSON('/api/v1/jobs/[% export_job_id \| html %]', function(job) {
		814	if (job.data.report.errors.length) {
		815	humanMsg.displayMsg(
		816	_("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
		817	{ className: 'humanError' }
		818	);
		819	}
		820	else {
		821	let export_links = Object.entries(job.data.report.export_links);
		822	let export_links_html = export_links.map(([format, href]) =>
		823	`<p>${format}: <a href=${href}>${href}</a></p>`
		824	).join('');
		825	if (export_links.length > 1) {
		826	export_links_html =
		827	`<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
		828	}
		829	$(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
		830	.appendTo("#job_callback");
		831	}
		832	});
		833	});
		834	[% END %]
		835
785	var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') \| html %]", 10);	836	var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') \| html %]", 10);
786	var q_array = new Array(); // will hold search terms, if present	837	var q_array = new Array(); // will hold search terms, if present
787	var PREF_LocalCoverImages = parseInt( "[% Koha.Preference('LocalCoverImages') \| html %]", 10);	838	var PREF_LocalCoverImages = parseInt( "[% Koha.Preference('LocalCoverImages') \| html %]", 10);





subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 74;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my @mappings = (
        {

            marc_type => 'marc21',
            marc_field => '650(avxyz)',
        },
        {
            name => 'local-number',
            type => 'string',
            facet => 0,
            suggestible => 0,
            searchable => 1,
            sort => 1,
            marc_type => 'marc21',
            marc_field => '999c',
        },
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');

        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
    );

    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->leader('     cam  22      a 4500');
    $marc_record_2->append_fields(

    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");

        MARC::Field->new('100', '', '', a => 'Author 1'),
        MARC::Field->new('110', '', '', a => 'Corp Author'),
        MARC::Field->new('210', '', '', a => 'Title 1'),
        # "|" is for testing escaping for multiple values with custom format
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large | record'),
        MARC::Field->new('999', '', '', c => '1234569'),
    );

    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');


    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');

    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");

    # Search export functionality
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
    my @es_response_docs;
    my $records_data;

    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {

        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);

        $docs = $see->marc_records_to_documents(\@source_docs);

        # Emulate Elasticsearch response docs structure
        @es_response_docs = map { { _source => $_ } } @{$docs};

        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');

        # $large_marc_record should not have been encoded as ISO2709
        # since exceeds maximum size, see above
        my @tmp = ($marc_record_1, $marc_record_2);
        is(
            $records_data->{ISO2709},
            join('', map { $_->as_usmarc() } @tmp),
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
        );

        my $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            MARC::File::XML::record($large_marc_record, 'MARC21'),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
        );

        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');

        $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
        );
    }

    my $custom_formats = <<'END';
- name: Biblionumbers
  fields: [local-number]
  multiple: ignore
- name: Title and author
  fields: [title, author]
  multiple: join
END
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');

    # Biblionumbers custom format
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
    # UTF-8 encode?
    is(
        $records_data,
        "1234567\n1234568\n1234569",
        "Records where correctly encoded for the custom format \"Biblionumbers\""
    );

    # Title and author custom format
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);

    my $encoded_data = join(
        "\n",
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
        "\"\",\"Author 2\"",
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
    );

    is(
        $records_data,
        $encoded_data,
        "Records where correctly encoded for the custom format \"Title and author\""
    );

    push @mappings, {
        name => 'title',
        type => 'string',


    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");

    plan tests => 5;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my $builder = t::lib::TestBuilder->new;
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
- 

Return to bug 27859

Lines 149-154 use C4::Koha qw( getitemtypeimagelocation GetAuthorisedValues ); Link Here

(-)a/catalogue/search.pl (+46 lines)
149	use URI::Escape;	149	use URI::Escape;
150	use POSIX qw(ceil floor);	150	use POSIX qw(ceil floor);
151	use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );	151	use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
		152	use Koha::BackgroundJob::SearchResultExport;
152		153
153	use Koha::ItemTypes;	154	use Koha::ItemTypes;
154	use Koha::Library::Groups;	155	use Koha::Library::Groups;
Lines 161-166 use Koha::SearchFilters; Link Here
161		162
162	use URI::Escape;	163	use URI::Escape;
163	use JSON qw( decode_json encode_json );	164	use JSON qw( decode_json encode_json );
		165	use Carp qw(croak);
164		166
165	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");	167	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
166	# create a new CGI object	168	# create a new CGI object
Lines 725-730 for (my $i=0;$i<@servers;$i++) { Link Here
725	} #/end of the for loop	727	} #/end of the for loop
726	#$template->param(FEDERATED_RESULTS => \@results_array);	728	#$template->param(FEDERATED_RESULTS => \@results_array);
727		729
		730	my $patron = Koha::Patrons->find( $borrowernumber );
		731	my $export_enabled =
		732	C4::Context->preference('EnableElasticsearchSearchResultExport') &&
		733	C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
		734	$patron && $patron->has_permission({ tools => 'export_catalog' });
		735
		736	$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
		737
		738	if ($export_enabled) {
		739
		740
		741	my $export = $cgi->param('export');
		742	my $preferred_format = $cgi->param('export_format');
		743	my $custom_export_formats = $searcher->search_result_export_custom_formats;
		744
		745	$template->param(custom_export_formats => $custom_export_formats);
		746
		747	# TODO: Need to handle $hits = 0?
		748	my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
		749
		750	if ($export && $preferred_format && $hits) {
		751	unless (
		752	$preferred_format eq 'ISO2709' \|\|
		753	$preferred_format eq 'MARCXML'
		754	) {
		755	if (!exists $custom_export_formats->{$preferred_format}) {
		756	croak "Invalid export format: $preferred_format";
		757	}
		758	else {
		759	$preferred_format = $custom_export_formats->{$preferred_format};
		760	}
		761	}
		762	my $size_limit = C4::Context->preference('SearchResultExportLimit') \|\| 0;
		763	my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
		764	my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
		765	my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
		766	size => $size,
		767	preferred_format => $preferred_format,
		768	elasticsearch_query => \%export_query
		769	});
		770	$template->param(export_job_id => $export_job_id);
		771	}
		772	}
		773
728	my $gotonumber = $cgi->param('gotoNumber');	774	my $gotonumber = $cgi->param('gotoNumber');
729	if ( $gotonumber && ( $gotonumber eq 'last' \|\| $gotonumber eq 'first' ) ) {	775	if ( $gotonumber && ( $gotonumber eq 'last' \|\| $gotonumber eq 'first' ) ) {
730	$template->{'VARS'}->{'gotoNumber'} = $gotonumber;	776	$template->{'VARS'}->{'gotoNumber'} = $gotonumber;

Lines 138-143 Searching: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
138	1: use	138	1: use
139	0: "don't use"	139	0: "don't use"
140	- 'the operator "phr" in the callnumber and standard number staff interface searches.'	140	- 'the operator "phr" in the callnumber and standard number staff interface searches.'
		141	-
		142	- pref: EnableElasticsearchSearchResultExport
		143	type: boolean
		144	default: yes
		145	choices:
		146	1: Enable
		147	0: Disable
		148	- Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
		149	-
		150	- pref: ElasticsearchSearchResultExportCustomFormats
		151	type: textarea
		152	syntax: text/x-yaml
		153	class: code
		154	- <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
		155	- <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
		156	- '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
		157	- '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
		158	- If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
		159	- '<p><strong>multiple</strong>: <i>ignore</i>\|<i>join</i>\|<i>newline</i></p>'
		160	- <p>The behavior when handling fields with multiple values.</p>
		161	- '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
		162	- '<p><i>join</i>: multiple values are concatenated using \"\|\" as a separator.</p>'
		163	- '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
		164	- 'Example:</br>'
		165	- '- name: Biblionumbers<br />'
		166	- '  fields: [local-number]<br />'
		167	- '  multiple: ignore<br />'
		168	- '- name: Title and author<br />'
		169	- '  fields: [title, author]<br />'
		170	- '  multiple: join<br /><br />'
		171	- '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
		172	-
		173	- Limit export from search results to a maximum of
		174	- pref: ElasticsearchSearchResultExportLimit
		175	class: integer
		176	- search result items (Elasticsearch only).<br /><br />
		177	- '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
141	Results display:	178	Results display:
142	-	179	-
143	- pref: numSearchResultsDropdown	180	- pref: numSearchResultsDropdown

Line 0 Link Here

(-)a/Koha/BackgroundJob/SearchResultExport.pm (+185 lines)
		1	package Koha::BackgroundJob::SearchResultExport;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	use Modern::Perl;
		19	use Try::Tiny;
		20	use Koha::SearchEngine::Search;
		21
		22	use File::Spec;
		23	use File::Path qw(mkpath);
		24	use Koha::Email;
		25	use Koha::UploadedFiles;
		26	use POSIX qw(strftime);
		27	use Digest::MD5 qw(md5_hex);
		28	use Carp qw(croak);
		29
		30	use base 'Koha::BackgroundJob';
		31
		32	=head1 NAME
		33
		34	Koha::BackgroundJob::SearchResultExport - Export data from search result
		35
		36	This is a subclass of Koha::BackgroundJob.
		37
		38	=head1 API
		39
		40	=head2 Class methods
		41
		42	=head3 job_type
		43
		44	Define the job type of this job: stage_marc_for_import
		45
		46	=cut
		47
		48	sub job_type {
		49	return 'search_result_export';
		50	}
		51
		52	=head3 process
		53
		54	Perform the export of search records.
		55
		56	=cut
		57
		58	sub process {
		59	my ( $self, $args ) = @_;
		60
		61	$self->start;
		62
		63	my $data = $self->decoded_data;
		64	my $borrowernumber = $data->{borrowernumber};
65	my $elasticsearch_query = $args->{elasticsearch_query};
66	my $preferred_format = $args->{preferred_format};
67	my $searcher = Koha::SearchEngine::Search->new({
68	index => $Koha::SearchEngine::BIBLIOS_INDEX
69	});
70	my $elasticsearch = $searcher->get_elasticsearch();
71
72	my $results = eval {
73	$elasticsearch->search(
74	index => $searcher->index_name,
75	scroll => '1m', #TODO: Syspref for scroll time limit?
76	size => 1000, #TODO: Syspref for batch size?
77	body => $elasticsearch_query
78	);
79	};
80	my @errors;
81	push @errors, $@ if $@;
82
83	my @docs;
84	my $encoded_results;
85	my %export_links;
86	my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
87
88	if (!@errors) {
89	my $scroll_id = $results->{_scroll_id};
90	while (@{$results->{hits}->{hits}}) {
91	push @docs, @{$results->{hits}->{hits}};
92	$self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
93	$results = $elasticsearch->scroll(
94	scroll => '1m',
95	scroll_id => $scroll_id
96	);
97	}
98
99	if ($preferred_format eq 'ISO2709' \|\| $preferred_format eq 'MARCXML') {
100	$encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
101	}
102	else {
103	$encoded_results->{$preferred_format->{name}} =
104	$searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
105	}
106
107	my %format_extensions = (
108	'ISO2709' => '.mrc',
109	'MARCXML' => '.xml',
110	);
111
112	my $upload_dir = Koha::UploadedFile->permanent_directory;
113
114	while (my ($format, $data) = each %{$encoded_results}) {
115	my $hash = md5_hex($data);
116	my $category = "search_marc_export";
117	my $time = strftime "%Y%m%d_%H%M", localtime time;
118	my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
119	my $filename = $category . '_' . $time . $ext;
120	my $file_dir = File::Spec->catfile($upload_dir, $category);
121	if ( !-d $file_dir) {
122	unless(mkpath $file_dir) {
123	push @errors, "Failed to create $file_dir";
124	next;
125	}
126	}
127	my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
128
129	my $fh = IO::File->new($filepath, "w");
130
131	if ($fh) {
132	$fh->binmode;
133	print $fh $data;
134	$fh->close;
135
136	my $size = -s $filepath;
137	my $file = Koha::UploadedFile->new({
138	hashvalue => $hash,
139	filename => $filename,
140	dir => $category,
141	filesize => $size,
142	owner => $borrowernumber,
143	uploadcategorycode => 'search_marc_export',
144	public => 0,
145	permanent => 1,
146	})->store;
147	my $id = $file->_result()->get_column('id');
148	$export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
149	}
150	else {
151	push @errors, "Failed to write \"$filepath\"";
152	}
153	}
154	}
155	my $report = {
156	export_links => \%export_links,
157	total => scalar @docs,
158	errors => \@errors,
159	query_string => $query_string,
160	};
161	$data->{report} = $report;
162	if (@errors) {
163	$self->set({ progress => 0, status => 'failed' })->store;
164	}
165	else {
166	$self->finish($data);
167	}
168	}
169
170	=head3 enqueue
171
172	Enqueue the new job
173
174	=cut
175
176	sub enqueue {
177	my ( $self, $args) = @_;
178	$self->SUPER::enqueue({
179	job_size => $args->{size},
180	job_args => $args,
181	job_queue => 'long_tasks',
182	});
183	}
184
185	1;

Lines 42-49 use YAML::XS; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-35 / +300 lines)
42		42
43	use List::Util qw( sum0 );	43	use List::Util qw( sum0 );
44	use MARC::File::XML;	44	use MARC::File::XML;
45	use MIME::Base64 qw( encode_base64 );	45	use MIME::Base64 qw(encode_base64 decode_base64);
46	use Encode qw( encode );	46	use Encode qw(encode decode);
47	use Business::ISBN;	47	use Business::ISBN;
48	use Scalar::Util qw( looks_like_number );	48	use Scalar::Util qw( looks_like_number );
49		49
Lines 542-548 sub marc_records_to_documents { Link Here
542	my $control_fields_rules = $rules->{control_fields};	542	my $control_fields_rules = $rules->{control_fields};
543	my $data_fields_rules = $rules->{data_fields};	543	my $data_fields_rules = $rules->{data_fields};
544	my $marcflavour = lc C4::Context->preference('marcflavour');	544	my $marcflavour = lc C4::Context->preference('marcflavour');
545	my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
546		545
547	my @record_documents;	546	my @record_documents;
548		547
Lines 742-776 sub marc_records_to_documents { Link Here
742	}	741	}
743	}	742	}
744	}	743	}
		744	my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
745		745
746	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	746	my ($encoded_record, $format) = $self->search_document_marc_record_encode(
747	$record->encoding('UTF-8');	747	$record,
748	if ($use_array) {	748	$preferred_format,
749	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);	749	$marcflavour
750	$record_document->{'marc_format'} = 'ARRAY';	750	);
		751
		752	if ($preferred_format eq 'ARRAY') {
		753	$record_document->{'marc_data_array'} = $encoded_record;
751	} else {	754	} else {
752	my @warnings;	755	$record_document->{'marc_data'} = $encoded_record;
753	{
754	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
755	local $SIG{__WARN__} = sub {
756	push @warnings, $_[0];
757	};
758	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
759	}
760	if (@warnings) {
761	# Suppress warnings if record length exceeded
762	unless (substr($record->leader(), 0, 5) eq '99999') {
763	foreach my $warning (@warnings) {
764	carp $warning;
765	}
766	}
767	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
768	$record_document->{'marc_format'} = 'MARCXML';
769	}
770	else {
771	$record_document->{'marc_format'} = 'base64ISO2709';
772	}
773	}	756	}
		757	$record_document->{'marc_format'} = $format;
		758
774		759
775	# Check if there is at least one available item	760	# Check if there is at least one available item
776	if ($self->index eq $BIBLIOS_INDEX) {	761	if ($self->index eq $BIBLIOS_INDEX) {
Lines 783-789 sub marc_records_to_documents { Link Here
783	onloan => undef,	768	onloan => undef,
784	itemlost => 0,	769	itemlost => 0,
785	})->count;	770	})->count;
786
787	$record_document->{available} = $avail_items ? \1 : \0;	771	$record_document->{available} = $avail_items ? \1 : \0;
788	}	772	}
789	}	773	}
Lines 793-798 sub marc_records_to_documents { Link Here
793	return \@record_documents;	777	return \@record_documents;
794	}	778	}
795		779
		780	=head2 search_document_marc_record_encode($record, $format, $marcflavour)
		781	my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
		782
		783	Encode a MARC::Record to the preferred marc document record format. If record
		784	exceeds ISO2709 maximum size record size and C<$format> is set to
		785	'base64ISO2709' format will fallback to 'MARCXML' instead.
		786
		787	=over 4
		788
		789	=item C<$record>
		790
		791	A MARC::Record object
		792
		793	=item C<$marcflavour>
		794
		795	The marcflavour to use
		796
		797	=back
		798
		799	=cut
		800
		801	sub search_document_marc_record_encode {
		802	my ($self, $record, $format, $marcflavour) = @_;
		803
		804	$record->encoding('UTF-8');
		805
		806	if ($format eq 'ARRAY') {
		807	return ($self->_marc_to_array($record), $format);
		808	}
		809	elsif ($format eq 'base64ISO2709' \|\| $format eq 'ISO2709') {
		810	my @warnings;
		811	my $marc_data;
		812	{
		813	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
		814	local $SIG{__WARN__} = sub {
		815	push @warnings, $_[0];
		816	};
		817	$marc_data = $record->as_usmarc();
		818	}
		819	if (@warnings) {
		820	# Suppress warnings if record length exceeded
		821	unless (substr($record->leader(), 0, 5) eq '99999') {
		822	foreach my $warning (@warnings) {
		823	carp $warning;
		824	}
		825	}
		826	return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
		827	}
		828	else {
		829	if ($format eq 'base64ISO2709') {
		830	$marc_data = encode_base64(encode('UTF-8', $marc_data));
		831	}
		832	return ($marc_data, $format);
		833	}
		834	}
		835	elsif ($format eq 'MARCXML') {
		836	return (MARC::File::XML::record($record, $marcflavour), $format);
		837	}
		838	else {
		839	# This should be unlikely to happen
		840	croak "Invalid marc record serialization format: $format";
		841	}
		842	}
		843
844	=head2 search_document_marc_record_decode
845	my $marc_record = $self->search_document_marc_record_decode(@result);
846
847	Extract marc data from Elasticsearch result and decode to MARC::Record object
848
849	=cut
850
851	sub search_document_marc_record_decode {
852	# Result is passed in as array, will get flattened
853	# and first element will be $result
854	my ($self, $result) = @_;
855	if ($result->{marc_format} eq 'base64ISO2709') {
856	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
857	}
858	elsif ($result->{marc_format} eq 'MARCXML') {
859	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
860	}
861	elsif ($result->{marc_format} eq 'ARRAY') {
862	return $self->_array_to_marc($result->{marc_data_array});
863	}
864	else {
865	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
866	}
867	}
868
869	=head2 search_documents_encode($docs, $preferred_format)
870
871	$records_data = $self->search_documents_encode($docs, $preferred_format)
872
873	Return marc encoded records from ElasticSearch search result documents. The return value
874	C<$marc_records> is a hashref with encoded records keyed by MARC format.
875
876	=over 4
877
878	=item C<$docs>
879
880	An arrayref of Elasticsearch search documents
881
882	=item C<$preferred_format>
883
884	The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
885	length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
886	is set to 'ISO2709'.
887
888	=back
889
890	=cut
891
892	sub search_documents_encode {
893
894	my ($self, $docs, $preferred_format) = @_;
895
896	my %encoded_records = (
897	'ISO2709' => [],
898	'MARCXML' => []
899	);
900
901	unless (exists $encoded_records{$preferred_format}) {
902	croak "Invalid preferred format: $preferred_format";
903	}
904
905	for my $es_record (@{$docs}) {
906	# Special optimized cases
907	my $marc_data;
908	my $resulting_format = $preferred_format;
909	if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
910	$marc_data = $es_record->{_source}{marc_data};
911	}
912	elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
913	$marc_data = decode_base64($es_record->{_source}->{marc_data});
914	}
915	else {
916	my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
917	my $marcflavour = lc C4::Context->preference('marcflavour');
918	($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
919	}
920	push @{$encoded_records{$resulting_format}}, $marc_data;
921	}
922	if (@{$encoded_records{'ISO2709'}}) {
923	$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
924	}
925	else {
926	delete $encoded_records{'ISO2709'};
927	}
928
929	if (@{$encoded_records{'MARCXML'}}) {
930	$encoded_records{'MARCXML'} = encode(
931	'UTF-8',
932	join(
933	"\n",
934	MARC::File::XML::header(),
935	join("\n", @{$encoded_records{'MARCXML'}}),
936	MARC::File::XML::footer()
937	)
938	);
939	}
940	else {
941	delete $encoded_records{'MARCXML'};
942	}
943
944	return \%encoded_records;
945	}
946
947	=head2 search_result_export_custom_formats()
948
949	$custom_formats = $self->search_result_export_custom_formats()
950
951	Return user defined custom search result export formats.
952
953	=cut
954
955	sub search_result_export_custom_formats {
956	my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') \|\| [];
957	my $custom_export_formats = {};
958
959	if (ref $export_custom_formats_pref eq 'ARRAY') {
960	for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
961	# TODO: Perhaps validate on save or trow error here instead of just
962	# ignoring invalid formats
963	my $format = $export_custom_formats_pref->[$i];
964	if (
965	ref $format->{fields} eq 'ARRAY' &&
966	@{$format->{fields}} &&
967	$format->{name}
968	) {
969	$format->{multiple} = 'ignore' unless exists $format->{multiple};
970	$custom_export_formats->{"custom_$i"} = $format;
971	}
972	}
973	}
974	return $custom_export_formats;
975	}
976
977	=head2 search_documents_custom_format_encode($docs, $custom_format)
978
979	$records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
980
981	Return encoded records from ElasticSearch search result documents using a
982	custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
983	Returns the encoded records.
984
985	=over 4
986
987	=item C<$docs>
988
989	An arrayref of Elasticsearch search documents
990
991	=item C<$format>
992
993	A hashref with the custom format definition.
994
995	=back
996
997	=cut
998
999	sub search_documents_custom_format_encode {
1000	my ($self, $docs, $format) = @_;
1001
1002	my $result;
1003
1004	my $doc_get_fields = sub {
1005	my ($doc, $fields) = @_;
1006	my @row;
1007	foreach my $field (@{$fields}) {
1008	my $values = $doc->{_source}->{$field};
1009	push @row, ref $values eq 'ARRAY' ? $values : [''];
1010	}
1011	return \@row;
1012	};
1013
1014	my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1015
1016	if($format->{multiple} eq 'ignore') {
1017	for (my $i = 0; $i < @rows; ++$i) {
1018	$rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1019	}
1020	}
1021	elsif($format->{multiple} eq 'newline') {
1022	if (@{$format->{fields}} == 1) {
1023	@rows = map { [join("\n", @{$_->[0]})] } @rows;
1024	}
1025	else {
1026	croak "'newline' is only valid for single field export formats";
1027	}
1028	}
1029	elsif($format->{multiple} eq 'join') {
1030	for (my $i = 0; $i < @rows; ++$i) {
1031	# Escape separator
1032	for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1033	for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1034	$rows[$i][$j][$k] =~ s/\\|/\\\|/g;
1035	}
1036	}
1037	# Separate multiple values with "\|"
1038	$rows[$i] = [map { join("\|", @{$_}) } @{$rows[$i]}];
1039	}
1040	}
1041	else {
1042	croak "Invalid 'multiple' option: " . $format->{multiple};
1043	}
1044	if (@{$format->{fields}} == 1) {
1045	@rows = grep { $_ ne '' } map { $_->[0] } @rows;
1046	}
1047	else {
1048	# Encode CSV
1049	for (my $i = 0; $i < @rows; ++$i) {
1050	# Escape quotes
1051	for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1052	$rows[$i][$j] =~ s/"/""/g;
1053	}
1054	$rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1055	}
1056	}
1057
1058	return encode('UTF-8', join("\n", @rows));
1059	}
1060
796	=head2 _marc_to_array($record)	1061	=head2 _marc_to_array($record)
797		1062
798	my @fields = _marc_to_array($record)	1063	my @fields = _marc_to_array($record)
Lines 864-881 sub _array_to_marc { Link Here
864	$record->leader($data->{leader});	1129	$record->leader($data->{leader});
865	for my $field (@{$data->{fields}}) {	1130	for my $field (@{$data->{fields}}) {
866	my $tag = (keys %{$field})[0];	1131	my $tag = (keys %{$field})[0];
867	$field = $field->{$tag};	1132	my $field_data = $field->{$tag};
868	my $marc_field;	1133	my $marc_field;
869	if (ref($field) eq 'HASH') {	1134	if (ref($field_data) eq 'HASH') {
870	my @subfields;	1135	my @subfields;
871	foreach my $subfield (@{$field->{subfields}}) {	1136	foreach my $subfield (@{$field_data->{subfields}}) {
872	my $code = (keys %{$subfield})[0];	1137	my $code = (keys %{$subfield})[0];
873	push @subfields, $code;	1138	push @subfields, $code;
874	push @subfields, $subfield->{$code};	1139	push @subfields, $subfield->{$code};
875	}	1140	}
876	$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);	1141	$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
877	} else {	1142	} else {
878	$marc_field = MARC::Field->new($tag, $field)	1143	$marc_field = MARC::Field->new($tag, $field_data)
879	}	1144	}
880	$record->append_fields($marc_field);	1145	$record->append_fields($marc_field);
881	}	1146	}

Lines 173-179 sub search_compat { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-28 / +3 lines)
173	my $index = $offset;	173	my $index = $offset;
174	my $hits = $results->{'hits'};	174	my $hits = $results->{'hits'};
175	foreach my $es_record (@{$hits->{'hits'}}) {	175	foreach my $es_record (@{$hits->{'hits'}}) {
176	$records[$index++] = $self->decode_record_from_result($es_record->{'_source'});	176	$records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
177	}	177	}
178		178
179	# consumers of this expect a name-spaced result, we provide the default	179	# consumers of this expect a name-spaced result, we provide the default
Lines 234-240 sub search_auth_compat { Link Here
234	# it's not reproduced here yet.	234	# it's not reproduced here yet.
235	my $authtype = $rs->single;	235	my $authtype = $rs->single;
236	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";	236	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
237	my $marc = $self->decode_record_from_result($record);	237	my $marc = $self->search_document_marc_record_decode($record);
238	my $mainentry = $marc->field($auth_tag_to_report);	238	my $mainentry = $marc->field($auth_tag_to_report);
239	my $reported_tag;	239	my $reported_tag;
240	if ($mainentry) {	240	if ($mainentry) {
Lines 354-360 sub simple_search_compat { Link Here
354	my @records;	354	my @records;
355	my $hits = $results->{'hits'};	355	my $hits = $results->{'hits'};
356	foreach my $es_record (@{$hits->{'hits'}}) {	356	foreach my $es_record (@{$hits->{'hits'}}) {
357	push @records, $self->decode_record_from_result($es_record->{'_source'});	357	push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
358	}	358	}
359	return (undef, \@records, $hits->{'total'});	359	return (undef, \@records, $hits->{'total'});
360	}	360	}
Lines 374-404 sub extract_biblionumber { Link Here
374	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );	374	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
375	}	375	}
376		376
377	=head2 decode_record_from_result
378	my $marc_record = $self->decode_record_from_result(@result);
379
380	Extracts marc data from Elasticsearch result and decodes to MARC::Record object
381
382	=cut
383
384	sub decode_record_from_result {
385	# Result is passed in as array, will get flattened
386	# and first element will be $result
387	my ( $self, $result ) = @_;
388	if ($result->{marc_format} eq 'base64ISO2709') {
389	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
390	}
391	elsif ($result->{marc_format} eq 'MARCXML') {
392	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
393	}
394	elsif ($result->{marc_format} eq 'ARRAY') {
395	return $self->_array_to_marc($result->{marc_data_array});
396	}
397	else {
398	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
399	}
400	}
401
402	=head2 max_result_window	377	=head2 max_result_window
403		378
404	Returns the maximum number of results that can be fetched	379	Returns the maximum number of results that can be fetched

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+23 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "27859",
	5	description => "Add system preferences",
	6	up => sub {
	7	my ($args) = @_;
	8	my ($dbh, $out) = @$args{qw(dbh out)};
	9
	10	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
	11	say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
	12
	13	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
	14	say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
	15
	16	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
	17	say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
	18
	19	$dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709\|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
	20	$dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
	21	say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
	22	},
	23	}

Lines 204-210 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +4 lines)
204	('EdifactLSQ', 'location', 'location\|ccode', 'Map EDI sequence code (GIR+LSQ) to Koha Item field', 'Choice'),	204	('EdifactLSQ', 'location', 'location\|ccode', 'Map EDI sequence code (GIR+LSQ) to Koha Item field', 'Choice'),
205	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),	205	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
206	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),	206	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
207	('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709\|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),	207	('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709\|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
208	('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),	208	('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
209	('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),	209	('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
210	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),	210	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
Lines 220-225 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
220	('EnableSearchHistory','0','','Enable or disable search history','YesNo'),	220	('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
221	('EnableItemGroups','0','','Enable the item groups feature','YesNo'),	221	('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
222	('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),	222	('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
		223	('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
223	('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),	224	('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
224	('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),	225	('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
225	('ERMModule', '0', NULL, 'Enable the e-resource management module', 'YesNo'),	226	('ERMModule', '0', NULL, 'Enable the e-resource management module', 'YesNo'),
Lines 640-645 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
640	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),	641	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),
641	('SearchLimitLibrary', 'homebranch', 'homebranch\|holdingbranch\|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),	642	('SearchLimitLibrary', 'homebranch', 'homebranch\|holdingbranch\|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
642	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),	643	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
		644	('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
		645	('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer'),
643	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),	646	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
644	('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),	647	('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
645	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),	648	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
	1	[% USE Koha %]
	2
	3	[% BLOCK report %]
	4	[% SET report = job.report %]
	5	[% IF report %]
	6	[% IF job.status == 'finished' %]
	7	<div class="dialog message">Search results export for the query "[% report.query_string \| html %]" completed successfully</div>
	8	<ul>
	9	<li>[% report.total \| html %] records exported</li>
	10	[% IF report.export_links.keys.size > 1 %]
	11	<li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
	12	[% END %]
	13	[% FOREACH format IN report.export_links.keys.sort %]
	14	<li>[% format \| $raw %]: <a href="[% report.export_links.$format \| $raw %]">[% report.export_links.$format \| html %]</a></li>
	15	[% END %]
	16	</ul>
	17	[% ELSE %]
	18	<div class="dialog error alert">Search results export for the query "[% report.query_string \| html %] failed with the following errors:</div>
	19	<ul>
	20	[% FOREACH error in report.errors %]
	21	<li>[% error \| html %]</li>
	22	[% END %]
	23	</ul>
	24	[% END %]
	25	[% END %]
	26	[% END %]
	27
	28	[% BLOCK detail %]
	29	[% END %]
	30
	31	[% BLOCK js %]
	32	[% END %]

Lines 184-193 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-9 / +109 lines)
184		184
185	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	185	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
186		186
187	plan tests => 63;	187	plan tests => 74;
188		188
189	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	189	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
190	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	190	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
191		191
192	my @mappings = (	192	my @mappings = (
193	{	193	{
Lines 380-385 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
380	marc_type => 'marc21',	380	marc_type => 'marc21',
381	marc_field => '650(avxyz)',	381	marc_field => '650(avxyz)',
382	},	382	},
		383	{
		384	name => 'local-number',
		385	type => 'string',
		386	facet => 0,
		387	suggestible => 0,
		388	searchable => 1,
		389	sort => 1,
		390	marc_type => 'marc21',
		391	marc_field => '999c',
		392	},
383	);	393	);
384		394
385	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');	395	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 427-432 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
427	MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),	437	MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
428	MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),	438	MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
429	);	439	);
		440
430	my $marc_record_2 = MARC::Record->new();	441	my $marc_record_2 = MARC::Record->new();
431	$marc_record_2->leader(' cam 22 a 4500');	442	$marc_record_2->leader(' cam 22 a 4500');
432	$marc_record_2->append_fields(	443	$marc_record_2->append_fields(
Lines 533-539 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
533	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');	544	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
534	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');	545	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
535		546
536	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	547	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
537		548
538	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");	549	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
539	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");	550	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 640-647 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
640	MARC::Field->new('100', '', '', a => 'Author 1'),	651	MARC::Field->new('100', '', '', a => 'Author 1'),
641	MARC::Field->new('110', '', '', a => 'Corp Author'),	652	MARC::Field->new('110', '', '', a => 'Corp Author'),
642	MARC::Field->new('210', '', '', a => 'Title 1'),	653	MARC::Field->new('210', '', '', a => 'Title 1'),
643	MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),	654	# "\|" is for testing escaping for multiple values with custom format
644	MARC::Field->new('999', '', '', c => '1234567'),	655	MARC::Field->new('245', '', '', a => 'Title:', b => 'large \| record'),
		656	MARC::Field->new('999', '', '', c => '1234569'),
645	);	657	);
646		658
647	my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');	659	my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
Lines 654-664 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
654		666
655	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');	667	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
656		668
657	$decoded_marc_record = $see->decode_record_from_result($docs->[0]);	669	$decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
658		670
659	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");	671	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
660	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");	672	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
661		673
		674	# Search export functionality
		675	# Koha::SearchEngine::Elasticsearch::search_documents_encode()
		676	my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
		677	my @es_response_docs;
		678	my $records_data;
		679
		680	for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
		681
		682	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
		683
		684	$docs = $see->marc_records_to_documents(\@source_docs);
		685
		686	# Emulate Elasticsearch response docs structure
		687	@es_response_docs = map { { _source => $_ } } @{$docs};
		688
		689	$records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
		690
		691	# $large_marc_record should not have been encoded as ISO2709
		692	# since exceeds maximum size, see above
		693	my @tmp = ($marc_record_1, $marc_record_2);
		694	is(
		695	$records_data->{ISO2709},
		696	join('', map { $_->as_usmarc() } @tmp),
		697	"ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
		698	);
		699
		700	my $expected_marc_xml = join("\n",
		701	MARC::File::XML::header(),
		702	MARC::File::XML::record($large_marc_record, 'MARC21'),
		703	MARC::File::XML::footer()
		704	);
		705
		706	is(
		707	$records_data->{MARCXML},
		708	$expected_marc_xml,
		709	"Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
		710	);
		711
		712	$records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
		713
		714	$expected_marc_xml = join("\n",
		715	MARC::File::XML::header(),
		716	join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
		717	MARC::File::XML::footer()
		718	);
		719
		720	is(
		721	$records_data->{MARCXML},
		722	$expected_marc_xml,
		723	"MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
		724	);
		725	}
		726
		727	my $custom_formats = <<'END';
		728	- name: Biblionumbers
		729	fields: [local-number]
		730	multiple: ignore
		731	- name: Title and author
		732	fields: [title, author]
		733	multiple: join
		734	END
		735	t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
		736	$custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
		737
738	# Biblionumbers custom format
739	$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
740	# UTF-8 encode?
741	is(
742	$records_data,
743	"1234567\n1234568\n1234569",
744	"Records where correctly encoded for the custom format \"Biblionumbers\""
745	);
746
747	# Title and author custom format
748	$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
749
750	my $encoded_data = join(
751	"\n",
752	"\"Title:\|first record\|Title: first record\",\"Author 1\|Corp Author\"",
753	"\"\",\"Author 2\"",
754	"\"Title:\|large \\\| record\|Title: large \\\| record\",\"Author 1\|Corp Author\""
755	);
756
757	is(
758	$records_data,
759	$encoded_data,
760	"Records where correctly encoded for the custom format \"Title and author\""
761	);
762
662	push @mappings, {	763	push @mappings, {
663	name => 'title',	764	name => 'title',
664	type => 'string',	765	type => 'string',
Lines 791-797 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
791		892
792	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');	893	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
793		894
794	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	895	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
795		896
796	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");	897	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
797	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");	898	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 802-808 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
802	plan tests => 5;	903	plan tests => 5;
803		904
804	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	905	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
805	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	906	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
806		907
807	my $builder = t::lib::TestBuilder->new;	908	my $builder = t::lib::TestBuilder->new;
808	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{	909	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
809	-