Attachment #163761 for bug #27859

View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All

Lines 429-434 sub core_types_to_classes { Link Here

(-)a/Koha/BackgroundJob.pm (+1 lines)
429	marc_import_commit_batch => 'Koha::BackgroundJob::MARCImportCommitBatch',	429	marc_import_commit_batch => 'Koha::BackgroundJob::MARCImportCommitBatch',
430	marc_import_revert_batch => 'Koha::BackgroundJob::MARCImportRevertBatch',	430	marc_import_revert_batch => 'Koha::BackgroundJob::MARCImportRevertBatch',
431	pseudonymize_statistic => 'Koha::BackgroundJob::PseudonymizeStatistic',	431	pseudonymize_statistic => 'Koha::BackgroundJob::PseudonymizeStatistic',
		432	search_result_export => 'Koha::BackgroundJob::SearchResultExport',
432	};	433	};
433	}	434	}
434		435




package Koha::BackgroundJob::SearchResultExport;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Try::Tiny;
use Koha::SearchEngine::Search;

use File::Spec;
use File::Path qw(mkpath);
use Koha::Email;
use Koha::UploadedFiles;
use POSIX qw(strftime);
use Digest::MD5 qw(md5_hex);
use Carp qw(croak);

use base 'Koha::BackgroundJob';

=head1 NAME

Koha::BackgroundJob::SearchResultExport - Export data from search result

This is a subclass of Koha::BackgroundJob.

=head1 API

=head2 Class methods

=head3 job_type

Define the job type of this job: stage_marc_for_import

=cut

sub job_type {
    return 'search_result_export';
}

=head3 process

Perform the export of search records.

=cut

sub process {
    my ( $self, $args ) = @_;

    $self->start;

    my $data = $self->decoded_data;
    my $borrowernumber = $data->{borrowernumber};
    my $elasticsearch_query = $args->{elasticsearch_query};
    my $preferred_format = $args->{preferred_format};
    my $searcher = Koha::SearchEngine::Search->new({
        index => $Koha::SearchEngine::BIBLIOS_INDEX
    });
    my $elasticsearch = $searcher->get_elasticsearch();

    my $results = eval {
        $elasticsearch->search(
            index => $searcher->index_name,
            scroll => '1m', #TODO: Syspref for scroll time limit?
            size => 1000,  #TODO: Syspref for batch size?
            body => $elasticsearch_query
        );
    };
    my @errors;
    push @errors, $@ if $@;

    my @docs;
    my $encoded_results;
    my %export_links;
    my $query_string = $elasticsearch_query->{query}->{query_string}->{query};

    if (!@errors) {
        my $scroll_id = $results->{_scroll_id};
        while (@{$results->{hits}->{hits}}) {
            push @docs, @{$results->{hits}->{hits}};
            $self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
            $results = $elasticsearch->scroll(
                scroll => '1m',
                scroll_id => $scroll_id
            );
        }

        if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
            $encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
        }
        else {
            $encoded_results->{$preferred_format->{name}} =
                $searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
        }

        my %format_extensions = (
            'ISO2709' => '.mrc',
            'MARCXML' => '.xml',
        );

        my $upload_dir = Koha::UploadedFile->permanent_directory;

        while (my ($format, $data) = each %{$encoded_results}) {
            my $hash = md5_hex($data);
            my $category = "search_marc_export";
            my $time = strftime "%Y%m%d_%H%M", localtime time;
            my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
            my $filename = $category . '_' . $time . $ext;
            my $file_dir = File::Spec->catfile($upload_dir, $category);
            if ( !-d $file_dir) {
                unless(mkpath $file_dir) {
                    push @errors, "Failed to create $file_dir";
                    next;
                }
            }
            my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");

            my $fh = IO::File->new($filepath, "w");

            if ($fh) {
                $fh->binmode;
                print $fh $data;
                $fh->close;

                my $size = -s $filepath;
                my $file = Koha::UploadedFile->new({
                        hashvalue => $hash,
                        filename  => $filename,
                        dir       => $category,
                        filesize  => $size,
                        owner     => $borrowernumber,
                        uploadcategorycode => 'search_marc_export',
                        public    => 0,
                        permanent => 1,
                    })->store;
                my $id = $file->_result()->get_column('id');
                $export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
            }
            else {
                push @errors, "Failed to write \"$filepath\"";
            }
        }
    }
    my $report = {
        export_links => \%export_links,
        total => scalar @docs,
        errors => \@errors,
        query_string => $query_string,
    };
    $data->{report}   = $report;
    if (@errors) {
        $self->set({ progress => 0, status => 'failed' })->store;
    }
    else {
        $self->finish($data);
    }
}

=head3 enqueue

Enqueue the new job

=cut

sub enqueue {
    my ( $self, $args) = @_;
    $self->SUPER::enqueue({
        job_size => $args->{size},
        job_args => $args,
        job_queue => 'long_tasks',
    });
}

1;





use List::Util qw( sum0 );
use MARC::File::XML;
use MIME::Base64 qw(encode_base64 decode_base64);
use Encode qw(encode decode);
use Business::ISBN;
use Scalar::Util qw( looks_like_number );


    my $control_fields_rules = $rules->{control_fields};
    my $data_fields_rules = $rules->{data_fields};
    my $marcflavour = lc C4::Context->preference('marcflavour');
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';

    my @record_documents;


                }
            }
        }
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');

        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
            $record,
            $preferred_format,
            $marcflavour
        );

        if ($preferred_format eq 'ARRAY') {
            $record_document->{'marc_data_array'} = $encoded_record;
        } else {
            $record_document->{'marc_data'} = $encoded_record;
            {
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
                local $SIG{__WARN__} = sub {
                    push @warnings, $_[0];
                };
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
            }
            if (@warnings) {
                # Suppress warnings if record length exceeded
                unless (substr($record->leader(), 0, 5) eq '99999') {
                    foreach my $warning (@warnings) {
                        carp $warning;
                    }
                }
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
                $record_document->{'marc_format'} = 'MARCXML';
            }
            else {
                $record_document->{'marc_format'} = 'base64ISO2709';
            }
        }
        $record_document->{'marc_format'} = $format;


        # Check if there is at least one available item
        if ($self->index eq $BIBLIOS_INDEX) {

                    onloan       => undef,
                    itemlost     => 0,
                })->count;

                $record_document->{available} = $avail_items ? \1 : \0;
            }
        }

    return \@record_documents;
}

=head2 search_document_marc_record_encode($record, $format, $marcflavour)
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)

Encode a MARC::Record to the preferred marc document record format. If record
exceeds ISO2709 maximum size record size and C<$format> is set to
'base64ISO2709' format will fallback to 'MARCXML' instead.

=over 4

=item C<$record>

A MARC::Record object

=item C<$marcflavour>

The marcflavour to use

=back

=cut

sub search_document_marc_record_encode {
    my ($self, $record, $format, $marcflavour) = @_;

    $record->encoding('UTF-8');

    if ($format eq 'ARRAY') {
        return ($self->_marc_to_array($record), $format);
    }
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
        my @warnings;
        my $marc_data;
        {
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
            local $SIG{__WARN__} = sub {
                push @warnings, $_[0];
            };
            $marc_data = $record->as_usmarc();
        }
        if (@warnings) {
            # Suppress warnings if record length exceeded
            unless (substr($record->leader(), 0, 5) eq '99999') {
                foreach my $warning (@warnings) {
                    carp $warning;
                }
            }
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
        }
        else {
            if ($format eq 'base64ISO2709') {
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
            }
            return ($marc_data, $format);
        }
    }
    elsif ($format eq 'MARCXML') {
        return (MARC::File::XML::record($record, $marcflavour), $format);
    }
    else {
        # This should be unlikely to happen
        croak "Invalid marc record serialization format: $format";
    }
}

=head2 search_document_marc_record_decode
    my $marc_record = $self->search_document_marc_record_decode(@result);

Extract marc data from Elasticsearch result and decode to MARC::Record object

=cut

sub search_document_marc_record_decode {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ($self, $result) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 search_documents_encode($docs, $preferred_format)

    $records_data = $self->search_documents_encode($docs, $preferred_format)

Return marc encoded records from ElasticSearch search result documents. The return value
C<$marc_records> is a hashref with encoded records keyed by MARC format.

=over 4

=item C<$docs>

An arrayref of Elasticsearch search documents

=item C<$preferred_format>

The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
is set to 'ISO2709'.

=back

=cut

sub search_documents_encode {

    my ($self, $docs, $preferred_format) = @_;

    my %encoded_records = (
        'ISO2709' => [],
        'MARCXML' => []
    );

    unless (exists $encoded_records{$preferred_format}) {
       croak "Invalid preferred format: $preferred_format";
    }

    for my $es_record (@{$docs}) {
        # Special optimized cases
        my $marc_data;
        my $resulting_format = $preferred_format;
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
            $marc_data = $es_record->{_source}{marc_data};
        }
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
            $marc_data = decode_base64($es_record->{_source}->{marc_data});
        }
        else {
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
            my $marcflavour = lc C4::Context->preference('marcflavour');
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
        }
        push @{$encoded_records{$resulting_format}}, $marc_data;
    }
    if (@{$encoded_records{'ISO2709'}}) {
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
    }
    else {
        delete $encoded_records{'ISO2709'};
    }

    if (@{$encoded_records{'MARCXML'}}) {
        $encoded_records{'MARCXML'} = encode(
            'UTF-8',
            join(
                "\n",
                MARC::File::XML::header(),
                join("\n", @{$encoded_records{'MARCXML'}}),
                MARC::File::XML::footer()
            )
        );
    }
    else {
        delete $encoded_records{'MARCXML'};
    }

    return \%encoded_records;
}

=head2 search_result_export_custom_formats()

    $custom_formats = $self->search_result_export_custom_formats()

Return user defined custom search result export formats.

=cut

sub search_result_export_custom_formats {
    my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') || [];
    my $custom_export_formats = {};

    if (ref $export_custom_formats_pref eq 'ARRAY') {
        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
            # TODO: Perhaps validate on save or trow error here instead of just
            # ignoring invalid formats
            my $format = $export_custom_formats_pref->[$i];
            if (
                ref $format->{fields} eq 'ARRAY' &&
                @{$format->{fields}} &&
                $format->{name}
            ) {
                $format->{multiple} = 'ignore' unless exists $format->{multiple};
                $custom_export_formats->{"custom_$i"} = $format;
            }
        }
    }
    return $custom_export_formats;
}

=head2 search_documents_custom_format_encode($docs, $custom_format)

    $records_data = $self->search_documents_custom_format_encode($docs, $custom_format)

Return encoded records from ElasticSearch search result documents using a
custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
Returns the encoded records.

=over 4

=item C<$docs>

An arrayref of Elasticsearch search documents

=item C<$format>

A hashref with the custom format definition.

=back

=cut

sub search_documents_custom_format_encode {
    my ($self, $docs, $format) = @_;

    my $result;

    my $doc_get_fields = sub {
        my ($doc, $fields) = @_;
        my @row;
        foreach my $field (@{$fields}) {
            my $values = $doc->{_source}->{$field};
            push @row, ref $values eq 'ARRAY' ? $values : [''];
        }
        return \@row;
    };

    my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};

    if($format->{multiple} eq 'ignore') {
        for (my $i = 0; $i < @rows; ++$i) {
            $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
        }
    }
    elsif($format->{multiple} eq 'newline') {
        if (@{$format->{fields}} == 1) {
            @rows = map { [join("\n", @{$_->[0]})] } @rows;
        }
        else {
            croak "'newline' is only valid for single field export formats";
        }
    }
    elsif($format->{multiple} eq 'join') {
        for (my $i = 0; $i < @rows; ++$i) {
            # Escape separator
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
                for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
                    $rows[$i][$j][$k] =~ s/\|/\\|/g;
                }
            }
            # Separate multiple values with "|"
            $rows[$i] = [map { join("|", @{$_}) } @{$rows[$i]}];
        }
    }
    else {
        croak "Invalid 'multiple' option: " . $format->{multiple};
    }
    if (@{$format->{fields}} == 1) {
        @rows = grep { $_ ne '' } map { $_->[0] } @rows;
    }
    else {
        # Encode CSV
        for (my $i = 0; $i < @rows; ++$i) {
            # Escape quotes
            for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
                $rows[$i][$j] =~ s/"/""/g;
            }
            $rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
        }
    }

    return encode('UTF-8', join("\n", @rows));
}

=head2 _marc_to_array($record)

    my @fields = _marc_to_array($record)

    $record->leader($data->{leader});
    for my $field (@{$data->{fields}}) {
        my $tag = (keys %{$field})[0];
        my $field_data = $field->{$tag};
        my $marc_field;
        if (ref($field_data) eq 'HASH') {
            my @subfields;
            foreach my $subfield (@{$field_data->{subfields}}) {
                my $code = (keys %{$subfield})[0];
                push @subfields, $code;
                push @subfields, $subfield->{$code};
            }
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
        } else {
            $marc_field = MARC::Field->new($tag, $field_data)
        }
        $record->append_fields($marc_field);
    }




    my $index = $offset;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
    }

    # consumers of this expect a name-spaced result, we provide the default

            # it's not reproduced here yet.
            my $authtype           = $rs->single;
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
            my $marc               = $self->search_document_marc_record_decode($record);
            my $mainentry          = $marc->field($auth_tag_to_report);
            my $reported_tag;
            if ($mainentry) {

    my @records;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
    }
    return (undef, \@records, $hits->{'total'});
}

    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
}

=head2 decode_record_from_result
    my $marc_record = $self->decode_record_from_result(@result);

Extracts marc data from Elasticsearch result and decodes to MARC::Record object

=cut

sub decode_record_from_result {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ( $self, $result ) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 max_result_window

Returns the maximum number of results that can be fetched




use URI::Escape;
use POSIX qw(ceil floor);
use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
use Koha::BackgroundJob::SearchResultExport;

use Koha::ItemTypes;
use Koha::Library::Groups;


use URI::Escape;
use JSON qw( decode_json encode_json );
use Carp qw(croak);

my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
# create a new CGI object

} #/end of the for loop
#$template->param(FEDERATED_RESULTS => \@results_array);

my $patron = Koha::Patrons->find( $borrowernumber );
my $export_enabled =
    C4::Context->preference('EnableElasticsearchSearchResultExport') &&
    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
    $patron && $patron->has_permission({ tools => 'export_catalog' });

$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';

if ($export_enabled) {


    my $export = $cgi->param('export');
    my $preferred_format = $cgi->param('export_format');
    my $custom_export_formats = $searcher->search_result_export_custom_formats;

    $template->param(custom_export_formats => $custom_export_formats);

    # TODO: Need to handle $hits = 0?
    my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;

    if ($export && $preferred_format && $hits) {
        unless (
            $preferred_format eq 'ISO2709' ||
            $preferred_format eq 'MARCXML'
        ) {
            if (!exists $custom_export_formats->{$preferred_format}) {
                croak "Invalid export format: $preferred_format";
            }
            else {
                $preferred_format = $custom_export_formats->{$preferred_format};
            }
        }
        my $size_limit = C4::Context->preference('SearchResultExportLimit') || 0;
        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
        my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
        my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
            size => $size,
            preferred_format => $preferred_format,
            elasticsearch_query => \%export_query
        });
        $template->param(export_job_id => $export_job_id);
    }
}

my $gotonumber = $cgi->param('gotoNumber');
if ( $gotonumber && ( $gotonumber eq 'last' || $gotonumber eq 'first' ) ) {
    $template->{'VARS'}->{'gotoNumber'} = $gotonumber;




use Modern::Perl;

return {
    bug_number => "27859",
    description => "Add system preferences",
    up => sub {
        my ($args) = @_;
        my ($dbh, $out) = @$args{qw(dbh out)};

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
        say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
        say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
        say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";

        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
        say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
    },
}




[% USE Koha %]

[% BLOCK report %]
    [% SET report = job.report %]
    [% IF report %]
        [% IF job.status == 'finished' %]
            <div class="dialog message">Search results export for the query "[% report.query_string | html %]" completed successfully</div>
            <ul>
              <li>[% report.total | html %] records exported</li>
              [% IF report.export_links.keys.size > 1 %]
                <li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
              [% END %]
              [% FOREACH format IN report.export_links.keys.sort %]
                <li>[% format | $raw %]: <a href="[% report.export_links.$format | $raw %]">[% report.export_links.$format | html %]</a></li>
              [% END %]
            </ul>
        [% ELSE %]
            <div class="dialog error alert">Search results export for the query "[% report.query_string | html %] failed with the following errors:</div>
            <ul>
            [% FOREACH error in report.errors %]
              <li>[% error | html %]</li>
            [% END %]
            </ul>
        [% END %]
    [% END %]
[% END %]

[% BLOCK detail %]
[% END %]

[% BLOCK js %]
[% END %]

Lines 238-243 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+4 lines)
238	'_id': 'marc_import_revert_batch',	238	'_id': 'marc_import_revert_batch',
239	'_str': _("Revert import MARC records")	239	'_str': _("Revert import MARC records")
240	},	240	},
		241	{
		242	'_id': 'search_result_export',
		243	'_str': _("Search result export")
		244	},
241	];	245	];
242		246
243	function get_job_type (job_type) {	247	function get_job_type (job_type) {

Lines 285-293 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
285	-	285	-
286	- "Elasticsearch MARC format: "	286	- "Elasticsearch MARC format: "
287	- pref: ElasticsearchMARCFormat	287	- pref: ElasticsearchMARCFormat
288	default: "ISO2709"	288	default: "base64ISO2709"
289	choices:	289	choices:
290	"ISO2709": "ISO2709 (exchange format)"	290	"base64ISO2709": "ISO2709 (exchange format)"
291	"ARRAY": "Searchable array"	291	"ARRAY": "Searchable array"
292	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.	292	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
293	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.	293	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.




                  1: use
                  0: "don't use"
            - 'the operator "phr" in the callnumber and standard number staff interface searches.'
        -
            - pref: EnableElasticsearchSearchResultExport
              type: boolean
              default: yes
              choices:
                  1: Enable
                  0: Disable
            - Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
        -
            - pref: ElasticsearchSearchResultExportCustomFormats
              type: textarea
              syntax: text/x-yaml
              class: code
            - <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
            - <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
            - '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
            - '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
            - If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
            - '<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i></p>'
            - <p>The behavior when handling fields with multiple values.</p>
            - '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
            - '<p><i>join</i>: multiple values are concatenated using \"|\" as a separator.</p>'
            - '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
            - 'Example:</br>'
            - '- name: Biblionumbers<br />'
            - '&nbsp;&nbsp;fields: [local-number]<br />'
            - '&nbsp;&nbsp;multiple: ignore<br />'
            - '- name: Title and author<br />'
            - '&nbsp;&nbsp;fields: [title, author]<br />'
            - '&nbsp;&nbsp;multiple: join<br /><br />'
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
        -
            - Limit export from search results to a maximum of
            - pref: ElasticsearchSearchResultExportLimit
              class: integer
            - search result items (Elasticsearch only).<br /><br />
            - '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
    Results display:
        -
            - pref: numSearchResultsDropdown

Lines 344-349 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+51 lines)
344	</div> <!-- /.btn-group -->	344	</div> <!-- /.btn-group -->
345	[% END %]	345	[% END %]
346		346
		347	[% IF export_enabled %]
		348	<div class="btn-group">
		349	<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
		350	Export all results<span class="caret"></span>
		351	</button>
		352	<ul class="dropdown-menu">
		353	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
		354	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
		355	[% FOREACH id IN custom_export_formats.keys %]
		356	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=[% id \| uri %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name \| html %]</a></li>
		357	[% END %]
		358	</ul>
		359	</div> <!-- /.btn-group -->
		360	[% END %]
		361
347	</div> <!-- /#selection_ops -->	362	</div> <!-- /#selection_ops -->
348	<form id="build_batch_record_modification" method="post" action="/cgi-bin/koha/tools/batch_record_modification.pl">	363	<form id="build_batch_record_modification" method="post" action="/cgi-bin/koha/tools/batch_record_modification.pl">
349	[% INCLUDE 'csrf-token.inc' %]	364	[% INCLUDE 'csrf-token.inc' %]
Lines 385-390 Link Here
385	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>	400	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>
386	[% END %]	401	[% END %]
387		402
		403	[% IF export_job_id %]
		404	<div class="dialog message">
		405	<p>Exporting records, the export will be processed as soon as possible.</p>
		406	[% INCLUDE "job_progress.inc" job_id=export_job_id %]
		407	<p><a class="job_details" href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=[% export_job_id \| uri %]" title="View detail of the enqueued job">View detail of the enqueued job</a>
		408	<div id="job_callback"></div>
		409	</div>
		410	[% END %]
		411
388	<!-- Search Results Table -->	412	<!-- Search Results Table -->
389	[% IF ( total ) %]	413	[% IF ( total ) %]
390	[% IF ( scan ) %]	414	[% IF ( scan ) %]
Lines 802-808 Link Here
802	[% Asset.css("css/humanmsg.css") \| $raw %]	826	[% Asset.css("css/humanmsg.css") \| $raw %]
803	[% Asset.js("lib/jquery/plugins/humanmsg.js") \| $raw %]	827	[% Asset.js("lib/jquery/plugins/humanmsg.js") \| $raw %]
804	[% INCLUDE 'select2.inc' %]	828	[% INCLUDE 'select2.inc' %]
		829	[% INCLUDE 'str/job_progress.inc' %]
		830	[% Asset.js("js/job_progress.js") \| $raw %]
805	<script>	831	<script>
		832	[% IF export_job_id %]
		833	updateProgress([% export_job_id \| html %], function() {
		834	$.getJSON('/api/v1/jobs/[% export_job_id \| html %]', function(job) {
		835	if (job.data.report.errors.length) {
		836	humanMsg.displayMsg(
		837	_("Export failed with the following errors: ") + "<br>" + job.data.report.errors.join('<br>'),
		838	{ className: 'humanError' }
		839	);
		840	}
		841	else {
		842	let export_links = Object.entries(job.data.report.export_links);
		843	let export_links_html = export_links.map(([format, href]) =>
		844	`<p>${format}: <a href=${href}>${href}</a></p>`
		845	).join('');
		846	if (export_links.length > 1) {
		847	export_links_html =
		848	`<p>${_("Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead")}</p>${export_links_html}`;
		849	}
		850	$(`<p>${_("Export finished successfully:")}</p>${export_links_html}`)
		851	.appendTo("#job_callback");
		852	}
		853	});
		854	});
		855	[% END %]
		856
806	var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') \| html %]", 10);	857	var PREF_AmazonCoverImages = parseInt( "[% Koha.Preference('AmazonCoverImages') \| html %]", 10);
807	var q_array = new Array(); // will hold search terms, if present	858	var q_array = new Array(); // will hold search terms, if present
808	var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') \| html %]", 10);	859	var PREF_IntranetCoce = parseInt( "[% Koha.Preference('IntranetCoce') \| html %]", 10);





subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 77;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my @mappings = (
        {

            marc_type => 'marc21',
            marc_field => '650(avxyz)',
        },
        {
            name => 'local-number',
            type => 'string',
            facet => 0,
            suggestible => 0,
            searchable => 1,
            sort => 1,
            marc_type => 'marc21',
            marc_field => '999c',
        },
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');

        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
    );

    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->leader('     cam  22      a 4500');
    $marc_record_2->append_fields(

    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");

        MARC::Field->new('100', '', '', a => 'Author 1'),
        MARC::Field->new('110', '', '', a => 'Corp Author'),
        MARC::Field->new('210', '', '', a => 'Title 1'),
        # "|" is for testing escaping for multiple values with custom format
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large | record'),
        MARC::Field->new('999', '', '', c => '1234569'),
    );

    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');


    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');

    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");

    # Search export functionality
    # Koha::SearchEngine::Elasticsearch::search_documents_encode()
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
    my @es_response_docs;
    my $records_data;

    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {

        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);

        $docs = $see->marc_records_to_documents(\@source_docs);

        # Emulate Elasticsearch response docs structure
        @es_response_docs = map { { _source => $_ } } @{$docs};

        $records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');

        # $large_marc_record should not have been encoded as ISO2709
        # since exceeds maximum size, see above
        my @tmp = ($marc_record_1, $marc_record_2);
        is(
            $records_data->{ISO2709},
            join('', map { $_->as_usmarc() } @tmp),
            "ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
        );

        my $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            MARC::File::XML::record($large_marc_record, 'MARC21'),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
        );

        $records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');

        $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
        );
    }

    my $custom_formats = <<'END';
- name: Biblionumbers
  fields: [local-number]
  multiple: ignore
- name: Title and author
  fields: [title, author]
  multiple: join
END
    t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
    $custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');

    # Biblionumbers custom format
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
    # UTF-8 encode?
    is(
        $records_data,
        "1234567\n1234568\n1234569",
        "Records where correctly encoded for the custom format \"Biblionumbers\""
    );

    # Title and author custom format
    $records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);

    my $encoded_data = join(
        "\n",
        "\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"",
        "\"\",\"Author 2\"",
        "\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\""
    );

    is(
        $records_data,
        $encoded_data,
        "Records where correctly encoded for the custom format \"Title and author\""
    );

    push @mappings, {
        name => 'title',
        type => 'string',


    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");

    plan tests => 5;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my $builder = t::lib::TestBuilder->new;
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
- 

Return to bug 27859

Lines 149-154 use C4::Koha qw( getitemtypeimagelocation GetAuthorisedValues ); Link Here

(-)a/catalogue/search.pl (+46 lines)
149	use URI::Escape;	149	use URI::Escape;
150	use POSIX qw(ceil floor);	150	use POSIX qw(ceil floor);
151	use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );	151	use C4::Search qw( searchResults enabled_staff_search_views z3950_search_args new_record_from_zebra );
		152	use Koha::BackgroundJob::SearchResultExport;
152		153
153	use Koha::ItemTypes;	154	use Koha::ItemTypes;
154	use Koha::Library::Groups;	155	use Koha::Library::Groups;
Lines 161-166 use Koha::SearchFilters; Link Here
161		162
162	use URI::Escape;	163	use URI::Escape;
163	use JSON qw( decode_json encode_json );	164	use JSON qw( decode_json encode_json );
		165	use Carp qw(croak);
164		166
165	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");	167	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
166	# create a new CGI object	168	# create a new CGI object
Lines 715-720 for (my $i=0;$i<@servers;$i++) { Link Here
715	} #/end of the for loop	717	} #/end of the for loop
716	#$template->param(FEDERATED_RESULTS => \@results_array);	718	#$template->param(FEDERATED_RESULTS => \@results_array);
717		719
		720	my $patron = Koha::Patrons->find( $borrowernumber );
		721	my $export_enabled =
		722	C4::Context->preference('EnableElasticsearchSearchResultExport') &&
		723	C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
		724	$patron && $patron->has_permission({ tools => 'export_catalog' });
		725
		726	$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
		727
		728	if ($export_enabled) {
		729
		730
		731	my $export = $cgi->param('export');
		732	my $preferred_format = $cgi->param('export_format');
		733	my $custom_export_formats = $searcher->search_result_export_custom_formats;
		734
		735	$template->param(custom_export_formats => $custom_export_formats);
		736
		737	# TODO: Need to handle $hits = 0?
		738	my $hits = $results_hashref->{biblioserver}->{'hits'} // 0;
		739
		740	if ($export && $preferred_format && $hits) {
		741	unless (
		742	$preferred_format eq 'ISO2709' \|\|
		743	$preferred_format eq 'MARCXML'
		744	) {
		745	if (!exists $custom_export_formats->{$preferred_format}) {
		746	croak "Invalid export format: $preferred_format";
		747	}
		748	else {
		749	$preferred_format = $custom_export_formats->{$preferred_format};
		750	}
		751	}
		752	my $size_limit = C4::Context->preference('SearchResultExportLimit') \|\| 0;
		753	my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
		754	my $size = $size_limit && $hits > $size_limit ? $size_limit : $hits;
		755	my $export_job_id = Koha::BackgroundJob::SearchResultExport->new->enqueue({
		756	size => $size,
		757	preferred_format => $preferred_format,
		758	elasticsearch_query => \%export_query
		759	});
		760	$template->param(export_job_id => $export_job_id);
		761	}
		762	}
		763
718	my $gotonumber = $cgi->param('gotoNumber');	764	my $gotonumber = $cgi->param('gotoNumber');
719	if ( $gotonumber && ( $gotonumber eq 'last' \|\| $gotonumber eq 'first' ) ) {	765	if ( $gotonumber && ( $gotonumber eq 'last' \|\| $gotonumber eq 'first' ) ) {
720	$template->{'VARS'}->{'gotoNumber'} = $gotonumber;	766	$template->{'VARS'}->{'gotoNumber'} = $gotonumber;

Lines 138-143 Searching: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+37 lines)
138	1: use	138	1: use
139	0: "don't use"	139	0: "don't use"
140	- 'the operator "phr" in the callnumber and standard number staff interface searches.'	140	- 'the operator "phr" in the callnumber and standard number staff interface searches.'
		141	-
		142	- pref: EnableElasticsearchSearchResultExport
		143	type: boolean
		144	default: yes
		145	choices:
		146	1: Enable
		147	0: Disable
		148	- Enable exporting search results. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (Elasticsearch only).
		149	-
		150	- pref: ElasticsearchSearchResultExportCustomFormats
		151	type: textarea
		152	syntax: text/x-yaml
		153	class: code
		154	- <p>Define custom export formats as a YAML list of associative arrays (Elasticsearch only).</p>
		155	- <p>Formats are defined using three properties, a required "<strong>name</strong>" and "<strong>fields</strong>" and an optional "<strong>multiple</strong>".</p>
		156	- '<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>'
		157	- '<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.'
		158	- If <strong>fields</strong> has a single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.</p>
		159	- '<p><strong>multiple</strong>: <i>ignore</i>\|<i>join</i>\|<i>newline</i></p>'
		160	- <p>The behavior when handling fields with multiple values.</p>
		161	- '<p><i>ignore</i>: the default option, only the first value is included, the rest ignored.</p>'
		162	- '<p><i>join</i>: multiple values are concatenated using \"\|\" as a separator.</p>'
		163	- '<p><i>newline</i>: a newline is inserted after each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>'
		164	- 'Example:</br>'
		165	- '- name: Biblionumbers<br />'
		166	- '  fields: [local-number]<br />'
		167	- '  multiple: ignore<br />'
		168	- '- name: Title and author<br />'
		169	- '  fields: [title, author]<br />'
		170	- '  multiple: join<br /><br />'
		171	- '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
		172	-
		173	- Limit export from search results to a maximum of
		174	- pref: ElasticsearchSearchResultExportLimit
		175	class: integer
		176	- search result items (Elasticsearch only).<br /><br />
		177	- '<p>See also: <a href="/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=EnableElasticsearchSearchResultExport">EnableElasticsearchSearchResultExport</a></p>'
141	Results display:	178	Results display:
142	-	179	-
143	- pref: numSearchResultsDropdown	180	- pref: numSearchResultsDropdown

Lines 186-195 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-9 / +109 lines)
186		186
187	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	187	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
188		188
189	plan tests => 66;	189	plan tests => 77;
190		190
191	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	191	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
192	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	192	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
193		193
194	my @mappings = (	194	my @mappings = (
195	{	195	{
Lines 393-398 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
393	marc_type => 'marc21',	393	marc_type => 'marc21',
394	marc_field => '650(avxyz)',	394	marc_field => '650(avxyz)',
395	},	395	},
		396	{
		397	name => 'local-number',
		398	type => 'string',
		399	facet => 0,
		400	suggestible => 0,
		401	searchable => 1,
		402	sort => 1,
		403	marc_type => 'marc21',
		404	marc_field => '999c',
		405	},
396	);	406	);
397		407
398	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');	408	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 441-446 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
441	MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),	451	MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
442	MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),	452	MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
443	);	453	);
		454
444	my $marc_record_2 = MARC::Record->new();	455	my $marc_record_2 = MARC::Record->new();
445	$marc_record_2->leader(' cam 22 a 4500');	456	$marc_record_2->leader(' cam 22 a 4500');
446	$marc_record_2->append_fields(	457	$marc_record_2->append_fields(
Lines 569-575 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
569	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');	580	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
570	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');	581	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
571		582
572	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	583	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
573		584
574	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");	585	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
575	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");	586	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 676-683 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
676	MARC::Field->new('100', '', '', a => 'Author 1'),	687	MARC::Field->new('100', '', '', a => 'Author 1'),
677	MARC::Field->new('110', '', '', a => 'Corp Author'),	688	MARC::Field->new('110', '', '', a => 'Corp Author'),
678	MARC::Field->new('210', '', '', a => 'Title 1'),	689	MARC::Field->new('210', '', '', a => 'Title 1'),
679	MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),	690	# "\|" is for testing escaping for multiple values with custom format
680	MARC::Field->new('999', '', '', c => '1234567'),	691	MARC::Field->new('245', '', '', a => 'Title:', b => 'large \| record'),
		692	MARC::Field->new('999', '', '', c => '1234569'),
681	);	693	);
682		694
683	my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');	695	my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
Lines 690-700 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
690		702
691	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');	703	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
692		704
693	$decoded_marc_record = $see->decode_record_from_result($docs->[0]);	705	$decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
694		706
695	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");	707	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
696	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");	708	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
697		709
		710	# Search export functionality
		711	# Koha::SearchEngine::Elasticsearch::search_documents_encode()
		712	my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
		713	my @es_response_docs;
		714	my $records_data;
		715
		716	for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
		717
		718	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
		719
		720	$docs = $see->marc_records_to_documents(\@source_docs);
		721
		722	# Emulate Elasticsearch response docs structure
		723	@es_response_docs = map { { _source => $_ } } @{$docs};
		724
		725	$records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709');
		726
		727	# $large_marc_record should not have been encoded as ISO2709
		728	# since exceeds maximum size, see above
		729	my @tmp = ($marc_record_1, $marc_record_2);
		730	is(
		731	$records_data->{ISO2709},
		732	join('', map { $_->as_usmarc() } @tmp),
		733	"ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
		734	);
		735
		736	my $expected_marc_xml = join("\n",
		737	MARC::File::XML::header(),
		738	MARC::File::XML::record($large_marc_record, 'MARC21'),
		739	MARC::File::XML::footer()
		740	);
		741
		742	is(
		743	$records_data->{MARCXML},
		744	$expected_marc_xml,
		745	"Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\""
		746	);
		747
		748	$records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML');
		749
		750	$expected_marc_xml = join("\n",
		751	MARC::File::XML::header(),
		752	join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
		753	MARC::File::XML::footer()
		754	);
		755
		756	is(
		757	$records_data->{MARCXML},
		758	$expected_marc_xml,
		759	"MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\""
		760	);
		761	}
		762
		763	my $custom_formats = <<'END';
		764	- name: Biblionumbers
		765	fields: [local-number]
		766	multiple: ignore
		767	- name: Title and author
		768	fields: [title, author]
		769	multiple: join
		770	END
		771	t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats);
		772	$custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats');
		773
774	# Biblionumbers custom format
775	$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]);
776	# UTF-8 encode?
777	is(
778	$records_data,
779	"1234567\n1234568\n1234569",
780	"Records where correctly encoded for the custom format \"Biblionumbers\""
781	);
782
783	# Title and author custom format
784	$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]);
785
786	my $encoded_data = join(
787	"\n",
788	"\"Title:\|first record\|Title: first record\",\"Author 1\|Corp Author\"",
789	"\"\",\"Author 2\"",
790	"\"Title:\|large \\\| record\|Title: large \\\| record\",\"Author 1\|Corp Author\""
791	);
792
793	is(
794	$records_data,
795	$encoded_data,
796	"Records where correctly encoded for the custom format \"Title and author\""
797	);
798
698	push @mappings, {	799	push @mappings, {
699	name => 'title',	800	name => 'title',
700	type => 'string',	801	type => 'string',
Lines 828-834 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
828		929
829	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');	930	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
830		931
831	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	932	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
832		933
833	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");	934	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
834	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");	935	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 839-845 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
839	plan tests => 5;	940	plan tests => 5;
840		941
841	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	942	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
842	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	943	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
843		944
844	my $builder = t::lib::TestBuilder->new;	945	my $builder = t::lib::TestBuilder->new;
845	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{	946	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
846	-

Line 0 Link Here

(-)a/Koha/BackgroundJob/SearchResultExport.pm (+185 lines)
		1	package Koha::BackgroundJob::SearchResultExport;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	use Modern::Perl;
		19	use Try::Tiny;
		20	use Koha::SearchEngine::Search;
		21
		22	use File::Spec;
		23	use File::Path qw(mkpath);
		24	use Koha::Email;
		25	use Koha::UploadedFiles;
		26	use POSIX qw(strftime);
		27	use Digest::MD5 qw(md5_hex);
		28	use Carp qw(croak);
		29
		30	use base 'Koha::BackgroundJob';
		31
		32	=head1 NAME
		33
		34	Koha::BackgroundJob::SearchResultExport - Export data from search result
		35
		36	This is a subclass of Koha::BackgroundJob.
		37
		38	=head1 API
		39
		40	=head2 Class methods
		41
		42	=head3 job_type
		43
		44	Define the job type of this job: stage_marc_for_import
		45
		46	=cut
		47
		48	sub job_type {
		49	return 'search_result_export';
		50	}
		51
		52	=head3 process
		53
		54	Perform the export of search records.
		55
		56	=cut
		57
		58	sub process {
		59	my ( $self, $args ) = @_;
		60
		61	$self->start;
		62
		63	my $data = $self->decoded_data;
		64	my $borrowernumber = $data->{borrowernumber};
65	my $elasticsearch_query = $args->{elasticsearch_query};
66	my $preferred_format = $args->{preferred_format};
67	my $searcher = Koha::SearchEngine::Search->new({
68	index => $Koha::SearchEngine::BIBLIOS_INDEX
69	});
70	my $elasticsearch = $searcher->get_elasticsearch();
71
72	my $results = eval {
73	$elasticsearch->search(
74	index => $searcher->index_name,
75	scroll => '1m', #TODO: Syspref for scroll time limit?
76	size => 1000, #TODO: Syspref for batch size?
77	body => $elasticsearch_query
78	);
79	};
80	my @errors;
81	push @errors, $@ if $@;
82
83	my @docs;
84	my $encoded_results;
85	my %export_links;
86	my $query_string = $elasticsearch_query->{query}->{query_string}->{query};
87
88	if (!@errors) {
89	my $scroll_id = $results->{_scroll_id};
90	while (@{$results->{hits}->{hits}}) {
91	push @docs, @{$results->{hits}->{hits}};
92	$self->progress( $self->progress + scalar @{$results->{hits}->{hits}} )->store;
93	$results = $elasticsearch->scroll(
94	scroll => '1m',
95	scroll_id => $scroll_id
96	);
97	}
98
99	if ($preferred_format eq 'ISO2709' \|\| $preferred_format eq 'MARCXML') {
100	$encoded_results = $searcher->search_documents_encode(\@docs, $preferred_format);
101	}
102	else {
103	$encoded_results->{$preferred_format->{name}} =
104	$searcher->search_documents_custom_format_encode(\@docs, $preferred_format);
105	}
106
107	my %format_extensions = (
108	'ISO2709' => '.mrc',
109	'MARCXML' => '.xml',
110	);
111
112	my $upload_dir = Koha::UploadedFile->permanent_directory;
113
114	while (my ($format, $data) = each %{$encoded_results}) {
115	my $hash = md5_hex($data);
116	my $category = "search_marc_export";
117	my $time = strftime "%Y%m%d_%H%M", localtime time;
118	my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
119	my $filename = $category . '_' . $time . $ext;
120	my $file_dir = File::Spec->catfile($upload_dir, $category);
121	if ( !-d $file_dir) {
122	unless(mkpath $file_dir) {
123	push @errors, "Failed to create $file_dir";
124	next;
125	}
126	}
127	my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
128
129	my $fh = IO::File->new($filepath, "w");
130
131	if ($fh) {
132	$fh->binmode;
133	print $fh $data;
134	$fh->close;
135
136	my $size = -s $filepath;
137	my $file = Koha::UploadedFile->new({
138	hashvalue => $hash,
139	filename => $filename,
140	dir => $category,
141	filesize => $size,
142	owner => $borrowernumber,
143	uploadcategorycode => 'search_marc_export',
144	public => 0,
145	permanent => 1,
146	})->store;
147	my $id = $file->_result()->get_column('id');
148	$export_links{$format} = "/cgi-bin/koha/tools/upload.pl?op=download&id=$id";
149	}
150	else {
151	push @errors, "Failed to write \"$filepath\"";
152	}
153	}
154	}
155	my $report = {
156	export_links => \%export_links,
157	total => scalar @docs,
158	errors => \@errors,
159	query_string => $query_string,
160	};
161	$data->{report} = $report;
162	if (@errors) {
163	$self->set({ progress => 0, status => 'failed' })->store;
164	}
165	else {
166	$self->finish($data);
167	}
168	}
169
170	=head3 enqueue
171
172	Enqueue the new job
173
174	=cut
175
176	sub enqueue {
177	my ( $self, $args) = @_;
178	$self->SUPER::enqueue({
179	job_size => $args->{size},
180	job_args => $args,
181	job_queue => 'long_tasks',
182	});
183	}
184
185	1;

Lines 42-49 use YAML::XS; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-35 / +300 lines)
42		42
43	use List::Util qw( sum0 );	43	use List::Util qw( sum0 );
44	use MARC::File::XML;	44	use MARC::File::XML;
45	use MIME::Base64 qw( encode_base64 );	45	use MIME::Base64 qw(encode_base64 decode_base64);
46	use Encode qw( encode );	46	use Encode qw(encode decode);
47	use Business::ISBN;	47	use Business::ISBN;
48	use Scalar::Util qw( looks_like_number );	48	use Scalar::Util qw( looks_like_number );
49		49
Lines 548-554 sub marc_records_to_documents { Link Here
548	my $control_fields_rules = $rules->{control_fields};	548	my $control_fields_rules = $rules->{control_fields};
549	my $data_fields_rules = $rules->{data_fields};	549	my $data_fields_rules = $rules->{data_fields};
550	my $marcflavour = lc C4::Context->preference('marcflavour');	550	my $marcflavour = lc C4::Context->preference('marcflavour');
551	my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
552		551
553	my @record_documents;	552	my @record_documents;
554		553
Lines 748-782 sub marc_records_to_documents { Link Here
748	}	747	}
749	}	748	}
750	}	749	}
		750	my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
751		751
752	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	752	my ($encoded_record, $format) = $self->search_document_marc_record_encode(
753	$record->encoding('UTF-8');	753	$record,
754	if ($use_array) {	754	$preferred_format,
755	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);	755	$marcflavour
756	$record_document->{'marc_format'} = 'ARRAY';	756	);
		757
		758	if ($preferred_format eq 'ARRAY') {
		759	$record_document->{'marc_data_array'} = $encoded_record;
757	} else {	760	} else {
758	my @warnings;	761	$record_document->{'marc_data'} = $encoded_record;
759	{
760	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
761	local $SIG{__WARN__} = sub {
762	push @warnings, $_[0];
763	};
764	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
765	}
766	if (@warnings) {
767	# Suppress warnings if record length exceeded
768	unless (substr($record->leader(), 0, 5) eq '99999') {
769	foreach my $warning (@warnings) {
770	carp $warning;
771	}
772	}
773	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
774	$record_document->{'marc_format'} = 'MARCXML';
775	}
776	else {
777	$record_document->{'marc_format'} = 'base64ISO2709';
778	}
779	}	762	}
		763	$record_document->{'marc_format'} = $format;
		764
780		765
781	# Check if there is at least one available item	766	# Check if there is at least one available item
782	if ($self->index eq $BIBLIOS_INDEX) {	767	if ($self->index eq $BIBLIOS_INDEX) {
Lines 789-795 sub marc_records_to_documents { Link Here
789	onloan => undef,	774	onloan => undef,
790	itemlost => 0,	775	itemlost => 0,
791	})->count;	776	})->count;
792
793	$record_document->{available} = $avail_items ? \1 : \0;	777	$record_document->{available} = $avail_items ? \1 : \0;
794	}	778	}
795	}	779	}
Lines 799-804 sub marc_records_to_documents { Link Here
799	return \@record_documents;	783	return \@record_documents;
800	}	784	}
801		785
		786	=head2 search_document_marc_record_encode($record, $format, $marcflavour)
		787	my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
		788
		789	Encode a MARC::Record to the preferred marc document record format. If record
		790	exceeds ISO2709 maximum size record size and C<$format> is set to
		791	'base64ISO2709' format will fallback to 'MARCXML' instead.
		792
		793	=over 4
		794
		795	=item C<$record>
		796
		797	A MARC::Record object
		798
		799	=item C<$marcflavour>
		800
		801	The marcflavour to use
		802
		803	=back
		804
		805	=cut
		806
		807	sub search_document_marc_record_encode {
		808	my ($self, $record, $format, $marcflavour) = @_;
		809
		810	$record->encoding('UTF-8');
		811
		812	if ($format eq 'ARRAY') {
		813	return ($self->_marc_to_array($record), $format);
		814	}
		815	elsif ($format eq 'base64ISO2709' \|\| $format eq 'ISO2709') {
		816	my @warnings;
		817	my $marc_data;
		818	{
		819	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
		820	local $SIG{__WARN__} = sub {
		821	push @warnings, $_[0];
		822	};
		823	$marc_data = $record->as_usmarc();
		824	}
		825	if (@warnings) {
		826	# Suppress warnings if record length exceeded
		827	unless (substr($record->leader(), 0, 5) eq '99999') {
		828	foreach my $warning (@warnings) {
		829	carp $warning;
		830	}
		831	}
		832	return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
		833	}
		834	else {
		835	if ($format eq 'base64ISO2709') {
		836	$marc_data = encode_base64(encode('UTF-8', $marc_data));
		837	}
		838	return ($marc_data, $format);
		839	}
		840	}
		841	elsif ($format eq 'MARCXML') {
		842	return (MARC::File::XML::record($record, $marcflavour), $format);
		843	}
		844	else {
		845	# This should be unlikely to happen
		846	croak "Invalid marc record serialization format: $format";
		847	}
		848	}
		849
850	=head2 search_document_marc_record_decode
851	my $marc_record = $self->search_document_marc_record_decode(@result);
852
853	Extract marc data from Elasticsearch result and decode to MARC::Record object
854
855	=cut
856
857	sub search_document_marc_record_decode {
858	# Result is passed in as array, will get flattened
859	# and first element will be $result
860	my ($self, $result) = @_;
861	if ($result->{marc_format} eq 'base64ISO2709') {
862	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
863	}
864	elsif ($result->{marc_format} eq 'MARCXML') {
865	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
866	}
867	elsif ($result->{marc_format} eq 'ARRAY') {
868	return $self->_array_to_marc($result->{marc_data_array});
869	}
870	else {
871	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
872	}
873	}
874
875	=head2 search_documents_encode($docs, $preferred_format)
876
877	$records_data = $self->search_documents_encode($docs, $preferred_format)
878
879	Return marc encoded records from ElasticSearch search result documents. The return value
880	C<$marc_records> is a hashref with encoded records keyed by MARC format.
881
882	=over 4
883
884	=item C<$docs>
885
886	An arrayref of Elasticsearch search documents
887
888	=item C<$preferred_format>
889
890	The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
891	length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
892	is set to 'ISO2709'.
893
894	=back
895
896	=cut
897
898	sub search_documents_encode {
899
900	my ($self, $docs, $preferred_format) = @_;
901
902	my %encoded_records = (
903	'ISO2709' => [],
904	'MARCXML' => []
905	);
906
907	unless (exists $encoded_records{$preferred_format}) {
908	croak "Invalid preferred format: $preferred_format";
909	}
910
911	for my $es_record (@{$docs}) {
912	# Special optimized cases
913	my $marc_data;
914	my $resulting_format = $preferred_format;
915	if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
916	$marc_data = $es_record->{_source}{marc_data};
917	}
918	elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
919	$marc_data = decode_base64($es_record->{_source}->{marc_data});
920	}
921	else {
922	my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
923	my $marcflavour = lc C4::Context->preference('marcflavour');
924	($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
925	}
926	push @{$encoded_records{$resulting_format}}, $marc_data;
927	}
928	if (@{$encoded_records{'ISO2709'}}) {
929	$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
930	}
931	else {
932	delete $encoded_records{'ISO2709'};
933	}
934
935	if (@{$encoded_records{'MARCXML'}}) {
936	$encoded_records{'MARCXML'} = encode(
937	'UTF-8',
938	join(
939	"\n",
940	MARC::File::XML::header(),
941	join("\n", @{$encoded_records{'MARCXML'}}),
942	MARC::File::XML::footer()
943	)
944	);
945	}
946	else {
947	delete $encoded_records{'MARCXML'};
948	}
949
950	return \%encoded_records;
951	}
952
953	=head2 search_result_export_custom_formats()
954
955	$custom_formats = $self->search_result_export_custom_formats()
956
957	Return user defined custom search result export formats.
958
959	=cut
960
961	sub search_result_export_custom_formats {
962	my $export_custom_formats_pref = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats') \|\| [];
963	my $custom_export_formats = {};
964
965	if (ref $export_custom_formats_pref eq 'ARRAY') {
966	for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
967	# TODO: Perhaps validate on save or trow error here instead of just
968	# ignoring invalid formats
969	my $format = $export_custom_formats_pref->[$i];
970	if (
971	ref $format->{fields} eq 'ARRAY' &&
972	@{$format->{fields}} &&
973	$format->{name}
974	) {
975	$format->{multiple} = 'ignore' unless exists $format->{multiple};
976	$custom_export_formats->{"custom_$i"} = $format;
977	}
978	}
979	}
980	return $custom_export_formats;
981	}
982
983	=head2 search_documents_custom_format_encode($docs, $custom_format)
984
985	$records_data = $self->search_documents_custom_format_encode($docs, $custom_format)
986
987	Return encoded records from ElasticSearch search result documents using a
988	custom format defined in the "ElasticsearchSearchResultExportCustomFormats" syspref.
989	Returns the encoded records.
990
991	=over 4
992
993	=item C<$docs>
994
995	An arrayref of Elasticsearch search documents
996
997	=item C<$format>
998
999	A hashref with the custom format definition.
1000
1001	=back
1002
1003	=cut
1004
1005	sub search_documents_custom_format_encode {
1006	my ($self, $docs, $format) = @_;
1007
1008	my $result;
1009
1010	my $doc_get_fields = sub {
1011	my ($doc, $fields) = @_;
1012	my @row;
1013	foreach my $field (@{$fields}) {
1014	my $values = $doc->{_source}->{$field};
1015	push @row, ref $values eq 'ARRAY' ? $values : [''];
1016	}
1017	return \@row;
1018	};
1019
1020	my @rows = map { $doc_get_fields->($_, $format->{fields}) } @{$docs};
1021
1022	if($format->{multiple} eq 'ignore') {
1023	for (my $i = 0; $i < @rows; ++$i) {
1024	$rows[$i] = [map { $_->[0] } @{$rows[$i]}];
1025	}
1026	}
1027	elsif($format->{multiple} eq 'newline') {
1028	if (@{$format->{fields}} == 1) {
1029	@rows = map { [join("\n", @{$_->[0]})] } @rows;
1030	}
1031	else {
1032	croak "'newline' is only valid for single field export formats";
1033	}
1034	}
1035	elsif($format->{multiple} eq 'join') {
1036	for (my $i = 0; $i < @rows; ++$i) {
1037	# Escape separator
1038	for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1039	for (my $k = 0; $k < @{$rows[$i][$j]}; ++$k) {
1040	$rows[$i][$j][$k] =~ s/\\|/\\\|/g;
1041	}
1042	}
1043	# Separate multiple values with "\|"
1044	$rows[$i] = [map { join("\|", @{$_}) } @{$rows[$i]}];
1045	}
1046	}
1047	else {
1048	croak "Invalid 'multiple' option: " . $format->{multiple};
1049	}
1050	if (@{$format->{fields}} == 1) {
1051	@rows = grep { $_ ne '' } map { $_->[0] } @rows;
1052	}
1053	else {
1054	# Encode CSV
1055	for (my $i = 0; $i < @rows; ++$i) {
1056	# Escape quotes
1057	for (my $j = 0; $j < @{$rows[$i]}; ++$j) {
1058	$rows[$i][$j] =~ s/"/""/g;
1059	}
1060	$rows[$i] = join(',', map { "\"$_\"" } @{$rows[$i]});
1061	}
1062	}
1063
1064	return encode('UTF-8', join("\n", @rows));
1065	}
1066
802	=head2 _marc_to_array($record)	1067	=head2 _marc_to_array($record)
803		1068
804	my @fields = _marc_to_array($record)	1069	my @fields = _marc_to_array($record)
Lines 870-887 sub _array_to_marc { Link Here
870	$record->leader($data->{leader});	1135	$record->leader($data->{leader});
871	for my $field (@{$data->{fields}}) {	1136	for my $field (@{$data->{fields}}) {
872	my $tag = (keys %{$field})[0];	1137	my $tag = (keys %{$field})[0];
873	$field = $field->{$tag};	1138	my $field_data = $field->{$tag};
874	my $marc_field;	1139	my $marc_field;
875	if (ref($field) eq 'HASH') {	1140	if (ref($field_data) eq 'HASH') {
876	my @subfields;	1141	my @subfields;
877	foreach my $subfield (@{$field->{subfields}}) {	1142	foreach my $subfield (@{$field_data->{subfields}}) {
878	my $code = (keys %{$subfield})[0];	1143	my $code = (keys %{$subfield})[0];
879	push @subfields, $code;	1144	push @subfields, $code;
880	push @subfields, $subfield->{$code};	1145	push @subfields, $subfield->{$code};
881	}	1146	}
882	$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);	1147	$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
883	} else {	1148	} else {
884	$marc_field = MARC::Field->new($tag, $field)	1149	$marc_field = MARC::Field->new($tag, $field_data)
885	}	1150	}
886	$record->append_fields($marc_field);	1151	$record->append_fields($marc_field);
887	}	1152	}

Lines 173-179 sub search_compat { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-28 / +3 lines)
173	my $index = $offset;	173	my $index = $offset;
174	my $hits = $results->{'hits'};	174	my $hits = $results->{'hits'};
175	foreach my $es_record (@{$hits->{'hits'}}) {	175	foreach my $es_record (@{$hits->{'hits'}}) {
176	$records[$index++] = $self->decode_record_from_result($es_record->{'_source'});	176	$records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
177	}	177	}
178		178
179	# consumers of this expect a name-spaced result, we provide the default	179	# consumers of this expect a name-spaced result, we provide the default
Lines 234-240 sub search_auth_compat { Link Here
234	# it's not reproduced here yet.	234	# it's not reproduced here yet.
235	my $authtype = $rs->single;	235	my $authtype = $rs->single;
236	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";	236	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
237	my $marc = $self->decode_record_from_result($record);	237	my $marc = $self->search_document_marc_record_decode($record);
238	my $mainentry = $marc->field($auth_tag_to_report);	238	my $mainentry = $marc->field($auth_tag_to_report);
239	my $reported_tag;	239	my $reported_tag;
240	if ($mainentry) {	240	if ($mainentry) {
Lines 354-360 sub simple_search_compat { Link Here
354	my @records;	354	my @records;
355	my $hits = $results->{'hits'};	355	my $hits = $results->{'hits'};
356	foreach my $es_record (@{$hits->{'hits'}}) {	356	foreach my $es_record (@{$hits->{'hits'}}) {
357	push @records, $self->decode_record_from_result($es_record->{'_source'});	357	push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
358	}	358	}
359	return (undef, \@records, $hits->{'total'});	359	return (undef, \@records, $hits->{'total'});
360	}	360	}
Lines 374-404 sub extract_biblionumber { Link Here
374	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );	374	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
375	}	375	}
376		376
377	=head2 decode_record_from_result
378	my $marc_record = $self->decode_record_from_result(@result);
379
380	Extracts marc data from Elasticsearch result and decodes to MARC::Record object
381
382	=cut
383
384	sub decode_record_from_result {
385	# Result is passed in as array, will get flattened
386	# and first element will be $result
387	my ( $self, $result ) = @_;
388	if ($result->{marc_format} eq 'base64ISO2709') {
389	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
390	}
391	elsif ($result->{marc_format} eq 'MARCXML') {
392	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
393	}
394	elsif ($result->{marc_format} eq 'ARRAY') {
395	return $self->_array_to_marc($result->{marc_data_array});
396	}
397	else {
398	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
399	}
400	}
401
402	=head2 max_result_window	377	=head2 max_result_window
403		378
404	Returns the maximum number of results that can be fetched	379	Returns the maximum number of results that can be fetched

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl (+23 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "27859",
	5	description => "Add system preferences",
	6	up => sub {
	7	my ($args) = @_;
	8	my ($dbh, $out) = @$args{qw(dbh out)};
	9
	10	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableElasticsearchSearchResultExport', 1, NULL, 'Enable search result export', 'YesNo') });
	11	say $out "Added new system preference 'EnableElasticsearchSearchResultExport'";
	12
	13	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea') });
	14	say $out "Added new system preference 'ElasticsearchSearchResultExportCustomFormats'";
	15
	16	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer') });
	17	say $out "Added new system preference 'ElasticsearchSearchResultExportLimit'";
	18
	19	$dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709\|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
	20	$dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
	21	say $out "Rename preference value in 'ElasticsearchMARCFormat' system preference";
	22	},
	23	}

Lines 223-229 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/mandatory/sysprefs.sql (-1 / +4 lines)
223	('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),	223	('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
224	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),	224	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
225	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),	225	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
226	('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709\|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),	226	('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709\|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
227	('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),	227	('EmailAddressForPatronRegistrations', '', '', ' If you choose EmailAddressForPatronRegistrations you have to enter a valid email address: ', 'free'),
228	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),	228	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
229	('EmailFieldPrecedence','email\|emailpro\|B_email','','Ordered list of patron email fields to use when AutoEmailPrimaryAddress is set to first valid','multiple'),	229	('EmailFieldPrecedence','email\|emailpro\|B_email','','Ordered list of patron email fields to use when AutoEmailPrimaryAddress is set to first valid','multiple'),
Lines 236-241 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
236	('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),	236	('EmailSMSSendDriverFromAddress', '', '', 'Email SMS send driver from address override', 'Free'),
237	('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),	237	('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
238	('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),	238	('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
		239	('EnableElasticsearchSearchResultExport', '1', '', 'Enable search result export', 'YesNo'),
239	('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),	240	('EnableExpiredPasswordReset', '0', NULL, 'Enable ability for patrons with expired password to reset their password directly', 'YesNo'),
240	('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),	241	('EnableItemGroupHolds','0','','Enable item groups holds feature','YesNo'),
241	('EnableItemGroups','0','','Enable the item groups feature','YesNo'),	242	('EnableItemGroups','0','','Enable the item groups feature','YesNo'),
Lines 678-683 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
678	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),	679	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),
679	('SearchLimitLibrary', 'homebranch', 'homebranch\|holdingbranch\|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),	680	('SearchLimitLibrary', 'homebranch', 'homebranch\|holdingbranch\|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
680	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),	681	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
		682	('ElasticsearchSearchResultExportCustomFormats', '', NULL, 'Search result export custom formats', 'textarea'),
		683	('ElasticsearchSearchResultExportLimit', NULL, NULL, 'Search result export limit', 'integer'),
681	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),	684	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
682	('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),	685	('SearchWithISSNVariations','0',NULL,'If enabled, search on all variations of the ISSN','YesNo'),
683	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),	686	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/search_result_export.inc (+32 lines)
	1	[% USE Koha %]
	2
	3	[% BLOCK report %]
	4	[% SET report = job.report %]
	5	[% IF report %]
	6	[% IF job.status == 'finished' %]
	7	<div class="dialog message">Search results export for the query "[% report.query_string \| html %]" completed successfully</div>
	8	<ul>
	9	<li>[% report.total \| html %] records exported</li>
	10	[% IF report.export_links.keys.size > 1 %]
	11	<li>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead</li>
	12	[% END %]
	13	[% FOREACH format IN report.export_links.keys.sort %]
	14	<li>[% format \| $raw %]: <a href="[% report.export_links.$format \| $raw %]">[% report.export_links.$format \| html %]</a></li>
	15	[% END %]
	16	</ul>
	17	[% ELSE %]
	18	<div class="dialog error alert">Search results export for the query "[% report.query_string \| html %] failed with the following errors:</div>
	19	<ul>
	20	[% FOREACH error in report.errors %]
	21	<li>[% error \| html %]</li>
	22	[% END %]
	23	</ul>
	24	[% END %]
	25	[% END %]
	26	[% END %]
	27
	28	[% BLOCK detail %]
	29	[% END %]
	30
	31	[% BLOCK js %]
	32	[% END %]