my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');

            return ($marc_data, $format);

use File::Spec;

use File::Path qw(mkpath);

use Koha::Email;

use Koha::UploadedFiles;

use POSIX qw(strftime);

use Digest::MD5 qw(md5_hex);

use Encode qw(encode);

use YAML::XS;

use Carp qw(croak);

my $patron = Koha::Patrons->find( $borrowernumber );

my $export_enabled =

    C4::Context->preference('EnableSearchResultMARCExport') &&

    C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&

    $patron && $patron->has_permission({ tools => 'export_catalog' });

$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';

if ($export_enabled) {

    my $export = $cgi->param('export');

    my $preferred_format = $cgi->param('export_format');

    my $export_custom_formats_pref = Load(C4::Context->preference('SearchResultMARCExportCustomFormats'));

    my $custom_export_formats = {};

    if (ref $export_custom_formats_pref eq 'ARRAY') {

        for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {

            # TODO: Validate on save or trow error here instead of just ignoreing

            my $format = $export_custom_formats_pref->[$i];

            if (

                ref $format->{fields} eq 'ARRAY' &&

                @{$format->{fields}} &&

                $format->{name}

            ) {

                $format->{multiple} = 'ignore' unless exists $format->{multiple};

                $custom_export_formats->{"custom_$i"} = $format;

            }

        }

    }

    $template->param(custom_export_formats => $custom_export_formats);

    if ($export && $preferred_format) {

        my $elasticsearch = $searcher->get_elasticsearch();

        my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0;

        my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};

        my $error;

        my $results = eval {

            $elasticsearch->search(

                index => $searcher->index_name,

                scroll => '1m', #TODO: Syspref for scroll time limit?

                size => 1000,  #TODO: Syspref for batch size?

                body => \%export_query

            );

        };

        if ($@) {

            $error = $@;

            $searcher->process_error($error);

        }

        my @docs;

        for my $doc (@{$results->{hits}->{hits}}) {

            push @docs, $doc;

        }

        my $scroll_id = $results->{_scroll_id};

        while (@{$results->{hits}->{hits}}) {

            $results = $elasticsearch->scroll(

                scroll => '1m',

                scroll_id => $scroll_id

            );

            for my $doc (@{$results->{hits}->{hits}}) {

                push @docs, $doc;

            }

        }

        my $message;

        my $export_links_html;

        if (!$error) {

            my $encoded_results = {};

            if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {

                $encoded_results = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);

            }

            elsif(exists $custom_export_formats->{$preferred_format}) {

                my $format = $custom_export_formats->{$preferred_format};

                my $result;

                my $doc_get_fields = sub {

                    my ($doc, $fields) = @_;

                    my @row;

                    foreach my $field (@{$fields}) {

                        my $values = $doc->{_source}->{$field};

                        push @row, $values && @{$values} ? $values : '';

                    }

                    return \@row;

                };

                my @rows = map { $doc_get_fields->($_, $format->{fields}) } @docs;

                if($format->{multiple} eq 'ignore') {

                    for (my $i = 0; $i < @rows; ++$i) {

                        $rows[$i] = [map { $_->[0] } @{$rows[$i]}];

                    }

                }

                elsif($format->{multiple} eq 'newline') {

                    if (@{$format->{fields}} == 1) {

                        @rows = map { [join("\n", @{$_->[0]})] } @rows;

                    }

                    else {

                        croak "'newline' is only valid for single field export formats";

                    }

                }

                elsif($format->{multiple} eq 'join') {

                    for (my $i = 0; $i < @rows; ++$i) {

                        $rows[$i] = [map { join("\t", @{$_}) } @{$rows[$i]}];

                    }

                }

                else {

                    croak "Invalid 'multiple' option: " . $format->{multiple};

                }

                if (@{$format->{fields}} == 1) {

                    @rows = map { $_->[0] } @rows;

                }

                else {

                    # Encode CSV

                    for (my $i = 0; $i < @rows; ++$i) {

                        $rows[$i] = join(',', map { $_ =~ s/"/""/; "\"$_\"" } @{$rows[$i]});

                    }

                }

                $encoded_results->{$format->{name}} = join("\n", @rows);

            }

            else {

                croak "Invalid export format: $preferred_format";

            }

            my %format_extensions = (

                'ISO2709' => '.mrc',

                'MARCXML' => '.xml',

            );

            my $upload_dir = Koha::UploadedFile->permanent_directory;

            my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";

            my %export_links;

            while (my ($format, $data) = each %{$encoded_results}) {

                $data = encode('UTF-8', $data);

                my $hash = md5_hex($data);

                my $category = "search_marc_export";

                my $time = strftime "%Y%m%d_%H%M", localtime time;

                my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';

                my $filename = $category . '_' . $time . $ext;

                my $file_dir = File::Spec->catfile($upload_dir, $category);

                if ( !-d $file_dir) {

                    mkpath $file_dir or die "Failed to create $file_dir";

                }

                my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");

                my $fh = IO::File->new($filepath, "w");

                if ($fh) {

                    $fh->binmode;

                    print $fh $data;

                    $fh->close;

                    my $size = -s $filepath;

                    my $file = Koha::UploadedFile->new({

                            hashvalue => $hash,

                            filename  => $filename,

                            dir       => $category,

                            filesize  => $size,

                            owner     => $borrowernumber,

                            uploadcategorycode => 'search_marc_export',

                            public    => 0,

                            permanent => 1,

                        })->store;

                    my $id = $file->_result()->get_column('id');

                    $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";

                }

                else {

                    croak "Failed to write \"$filepath\"";

                }

            }

            while (my ($format, $link) = each %export_links) {

                $export_links_html .= "$format: <a href=\"$link\">$link</a>\n";

            }

            my $query_string = $query->{query}->{query_string}->{query};

            my $links_count = keys %export_links;

            $export_links_html = $links_count > 1 ?

            "<p>Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead.</p>" . $export_links_html : $export_links_html;

            my $send_email = C4::Context->preference('SearchResultMARCExportEmail');

            if ($send_email) {

                if ($patron->email) {

                    my $export_from_address = C4::Context->preference('SearchResultMARCExportEmailFromAddress');

                    my $export_user_email = $patron->email;

                    my $mail = Koha::Email->create({

                            to => $export_user_email,

                            from => $export_from_address,

                            subject => "Marc export for query: $query_string",

                            html_body => $export_links_html,

                        });

                    $mail->send_or_die({ transport => $patron->library->smtp_server->transport });

                    $export_links_html .= "<p>An email has been sent to: $export_user_email</p>";

                }

                else {

                    $export_links_html .= "<p>Unable to send mail, the current user has no email address set</p>";

                }

            }

            $message = "<p>The export finished successfully:</p>" . $export_links_html;

            $template->param(export_message => $message);

        }

        else {

            $message = "An error occurred during marc export: $error",

            $template->param(export_error => $message);

        }

    }

}

use Modern::Perl;

return {

    bug_number => "27859",

    description => "Add system preferences",

    up => sub {

        my ($args) = @_;

        my ($dbh, $out) = @$args{qw(dbh out)};

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea') });

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo') });

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short') });

        $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });

        $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });

        $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });

        say $out "System preferences added";

    },

}

('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),

('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea'),

('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo'),

('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short'),

('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),

        -

            - pref: EnableSearchResultMARCExport

              type: boolean

              default: yes

              choices:

                  yes: Enable

                  no: Disable

            - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."

        -

            - pref: SearchResultMARCExportCustomFormats

              type: textarea

              syntax: text/x-yaml

              class: code

            - "Define custom export formats as a YAML list of associative arrays."

            - "A format have the required properties \"<strong>name</strong>\", \"<strong>fields</strong>\" and an optional \"<strong>multiple</strong>\".<br />"

            - "<p><strong>name</strong>: the human readable name of the format exposed in the staff interface.</p>"

            - "<p><strong>fields</strong>: a list of Elasticsearch fields to be included in the export.</p>"

            - "If <strong>fields</strong> contain a only single field the export result will contain one value per row, for multiple fields a CSV-file will be produced.<br />"

            - "<p><strong>multiple</strong>: <i>ignore</i>|<i>join</i>|<i>newline</i><br />The behavior when handling fields with multiple values.</p>"

            - "<p><i>ignore</i> is the default option, only the first value will be included, the rest ignored.</p>"

            - "<p><i>join</i>, multiple values will be contatenated with tab as a separator.</p>"

            - "<p><i>newline</i>, a newline will be inserted for each value. This option does not allow \"<strong>fields</strong>\" to contain multiple fields.</p>"

            - "Example:</br>"

            - "- name: Biblionumbers<br />"

            - "&nbsp;&nbsp;fields: [local-number]<br />"

            - "&nbsp;&nbsp;&nbsp;&nbsp;multiple: ignore<br />"

            - "- name: Title and author<br />"

            - "&nbsp;&nbsp;fields: [title, author]<br />"

            - "&nbsp;&nbsp;&nbsp;&nbsp;multiple: join<br />"

        -

            - "Limit exported MARC records from search results to a maximum of"

            - pref: SearchResultMARCExportLimit

              class: integer

            - "records."

        -

            - "Send an email with the export results to the current user"

            - pref: SearchResultMARCExportEmail

              type: boolean

              default: no

              choices:

                  1: Yes

                  0: No

        -

            - "Use the from-address"

            - pref: SearchResultMARCExportFromAddress

              class: short

            - "when mailing search results exports."

                            [% IF export_enabled %]

                                <div class="btn-group">

                                    <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">

                                        Export all results<span class="caret"></span>

                                    </button>

                                    <ul class="dropdown-menu">

                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>

                                        <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>

                                        [% FOREACH id IN custom_export_formats.keys %]

                                            <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>

                                        [% END %]

                                   </ul>

                                </div> <!-- /.btn-group -->

                            [% END %]

                [% IF ( export_message ) %]

                    <div class="dialog message">[% export_message %]</div>

                [% END %]

                [% IF ( export_error ) %]

                    <div class="dialog error">[% export_error %]</div>

                [% END %]

    # Search export functionality

    # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()

    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);

    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {

        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);

        $docs = $see->marc_records_to_documents(\@source_docs);

        # Emulate Elasticsearch response docs structure

        my @es_response_docs = map { { _source => $_ } } @{$docs};

        my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');

        # $large_marc_record should not have been encoded as ISO2709

        # since exceeds maximum size, see above

        my @tmp = ($marc_record_1, $marc_record_2);

        is(

            $records_data->{ISO2709},

            join('', map { $_->as_usmarc() } @tmp),

            "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""

        );

        my $expected_marc_xml = join("\n",

            MARC::File::XML::header(),

            MARC::File::XML::record($large_marc_record, 'MARC21'),

            MARC::File::XML::footer()

        );

        is(

            $records_data->{MARCXML},

            $expected_marc_xml,

            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""

        );

        $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');

        $expected_marc_xml = join("\n",

            MARC::File::XML::header(),

            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),

            MARC::File::XML::footer()

        );

        is(

            $records_data->{MARCXML},

            $expected_marc_xml,

            "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""

        );

    }