Attachment 132993 Details for Bug 27859 – Bug 27859: marc search result export

[patch] Bug 27859: marc search result export

Bug-27859-marc-search-result-export.patch (text/plain), 41.32 KB, created by David Gustafsson on 2022-04-06 12:22:46 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Gustafsson

Created: 2022-04-06 12:22:46 UTC

Size: 41.32 KB

patch

obsolete

>From 94278e99cee2e82f2fccf02401bea80e13e9a01d Mon Sep 17 00:00:00 2001
>From: David Gustafsson <david.gustafsson@ub.gu.se>
>Date: Fri, 25 Sep 2020 16:07:41 +0200
>Subject: [PATCH] Bug 27859: marc search result export
>
>Enable export of staff interface search results in different marc formats.
>The export will be mailed to the configured mail address of the current user.
>This feature is Elasticsearch only.
>
>To test:
>1) Apply patch
>2) Run installer/data/mysql/updatedatabase.pl
>3) Make sure the syspref EnableSearchResultMARCExport is enabled
>4) Make sure the syspref EnableSearchResultMARCExportEmail is enabled
>5) Make sure the current user has the tools -> export_catalog permission
>6) Make sure the current user has an email under your control set
>7) Perform a search
>8) Export the serach result by choosing a format under the
> "Export results" drop down
>9) Verify that link(s) with exported data has been mailed to the
> provided email
>10) Revoke the permission in 3) and make sure exporting is no longer
> possible
>11) Run tests in t/db_dependent/Koha/SearchEngine/Elasticsearch.t
>
>Sponsored-by: Gothenburg University Library
>---
> Koha/SearchEngine/Elasticsearch.pm | 214 ++++++++++++++---
> Koha/SearchEngine/Elasticsearch/Search.pm | 32 +--
> catalogue/search.pl | 227 ++++++++++++++++++
> ...able_search_result_marc_export_sysprefs.pl | 18 ++
> installer/data/mysql/mandatory/sysprefs.sql | 7 +-
> .../en/modules/admin/preferences/admin.pref | 4 +-
> .../modules/admin/preferences/searching.pref | 47 ++++
> .../prog/en/modules/catalogue/results.tt | 22 ++
> .../Koha/SearchEngine/Elasticsearch.t | 64 ++++-
> 9 files changed, 564 insertions(+), 71 deletions(-)
> create mode 100755 installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 85d8cba798..d5454cdb29 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -41,8 +41,8 @@ use YAML::XS;
> 
> use List::Util qw( sum0 );
> use MARC::File::XML;
>-use MIME::Base64 qw( encode_base64 );
>-use Encode qw( encode );
>+use MIME::Base64 qw(encode_base64 decode_base64);
>+use Encode qw(encode decode);
> use Business::ISBN;
> use Scalar::Util qw( looks_like_number );
> 
>@@ -541,7 +541,6 @@ sub marc_records_to_documents {
> my $control_fields_rules = $rules->{control_fields};
> my $data_fields_rules = $rules->{data_fields};
> my $marcflavour = lc C4::Context->preference('marcflavour');
>- my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
> 
> my @record_documents;
> 
>@@ -741,38 +740,187 @@ sub marc_records_to_documents {
> }
> }
> }
>+ my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
> 
>- # TODO: Perhaps should check if $records_document non empty, but really should never be the case
>- $record->encoding('UTF-8');
>- if ($use_array) {
>- $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
>- $record_document->{'marc_format'} = 'ARRAY';
>+ my ($encoded_record, $format) = $self->search_document_marc_record_encode(
>+ $record,
>+ $preferred_format,
>+ $marcflavour
>+ );
>+
>+ if ($preferred_format eq 'ARRAY') {
>+ $record_document->{'marc_data_array'} = $encoded_record;
> } else {
>- my @warnings;
>- {
>- # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
>- local $SIG{__WARN__} = sub {
>- push @warnings, $_[0];
>- };
>- $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
>- }
>- if (@warnings) {
>- # Suppress warnings if record length exceeded
>- unless (substr($record->leader(), 0, 5) eq '99999') {
>- foreach my $warning (@warnings) {
>- carp $warning;
>- }
>+ $record_document->{'marc_data'} = $encoded_record;
>+ }
>+ $record_document->{'marc_format'} = $format;
>+
>+ push @record_documents, $record_document;
>+ }
>+ return \@record_documents;
>+}
>+
>+=head2 search_document_marc_record_encode($record, $format, $marcflavour)
>+ my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
>+
>+Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
>+size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.
>+
>+=over 4
>+
>+=item C<$record>
>+
>+A MARC::Record object
>+
>+=item C<$marcflavour>
>+
>+The marcflavour to use
>+
>+=back
>+
>+=cut
>+
>+sub search_document_marc_record_encode {
>+ my ($self, $record, $format, $marcflavour) = @_;
>+
>+ $record->encoding('UTF-8');
>+
>+ if ($format eq 'ARRAY') {
>+ return ($self->_marc_to_array($record), $format);
>+ }
>+ elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
>+ my @warnings;
>+ my $marc_data;
>+ {
>+ # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
>+ local $SIG{__WARN__} = sub {
>+ push @warnings, $_[0];
>+ };
>+ $marc_data = $record->as_usmarc();
>+ }
>+ if (@warnings) {
>+ # Suppress warnings if record length exceeded
>+ unless (substr($record->leader(), 0, 5) eq '99999') {
>+ foreach my $warning (@warnings) {
>+ carp $warning;
> }
>- $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
>- $record_document->{'marc_format'} = 'MARCXML';
> }
>- else {
>- $record_document->{'marc_format'} = 'base64ISO2709';
>+ return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
>+ }
>+ else {
>+ if ($format eq 'base64ISO2709') {
>+ $marc_data = encode_base64(encode('UTF-8', $marc_data));
> }
>+ return ($marc_data, $format);
> }
>- push @record_documents, $record_document;
> }
>- return \@record_documents;
>+ elsif ($format eq 'MARCXML') {
>+ return (MARC::File::XML::record($record, $marcflavour), $format);
>+ }
>+ else {
>+ # This should be unlikely to happen
>+ croak "Invalid marc record serialization format: $format";
>+ }
>+}
>+
>+=head2 search_document_marc_record_decode
>+ my $marc_record = $self->search_document_marc_record_decode(@result);
>+
>+Extract marc data from Elasticsearch result and decode to MARC::Record object
>+
>+=cut
>+
>+sub search_document_marc_record_decode {
>+ # Result is passed in as array, will get flattened
>+ # and first element will be $result
>+ my ($self, $result) = @_;
>+ if ($result->{marc_format} eq 'base64ISO2709') {
>+ return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
>+ }
>+ elsif ($result->{marc_format} eq 'MARCXML') {
>+ return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
>+ }
>+ elsif ($result->{marc_format} eq 'ARRAY') {
>+ return $self->_array_to_marc($result->{marc_data_array});
>+ }
>+ else {
>+ Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
>+ }
>+}
>+
>+=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)
>+
>+ $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)
>+
>+Return marc encoded records from ElasticSearch search result documents. The return value
>+C<$marc_records> is a hashref with encoded records keyed by MARC format.
>+
>+=over 4
>+
>+=item C<$docs>
>+
>+An arrayref of Elasticsearch search documents
>+
>+=item C<$preferred_format>
>+
>+The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
>+length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
>+is set to 'ISO2709'.
>+
>+=back
>+
>+=cut
>+
>+sub search_document_marc_records_encode_from_docs {
>+
>+ my ($self, $docs, $preferred_format) = @_;
>+
>+ my %encoded_records = (
>+ 'ISO2709' => [],
>+ 'MARCXML' => []
>+ );
>+
>+ unless (exists $encoded_records{$preferred_format}) {
>+ croak "Invalid preferred format: $preferred_format";
>+ }
>+
>+ for my $es_record (@{$docs}) {
>+ # Special optimized cases
>+ my $marc_data;
>+ my $resulting_format = $preferred_format;
>+ if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
>+ $marc_data = $es_record->{_source}{marc_data};
>+ }
>+ elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
>+ # Stored as UTF-8 encoded binary in index, so needs to be decoded
>+ $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
>+ }
>+ else {
>+ my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
>+ my $marcflavour = lc C4::Context->preference('marcflavour');
>+ ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
>+ }
>+ push @{$encoded_records{$resulting_format}}, $marc_data;
>+ }
>+ if (@{$encoded_records{'ISO2709'}}) {
>+ $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
>+ }
>+ else {
>+ delete $encoded_records{'ISO2709'};
>+ }
>+
>+ if (@{$encoded_records{'MARCXML'}}) {
>+ $encoded_records{'MARCXML'} = join("\n",
>+ MARC::File::XML::header(),
>+ join("\n", @{$encoded_records{'MARCXML'}}),
>+ MARC::File::XML::footer()
>+ );
>+ }
>+ else {
>+ delete $encoded_records{'MARCXML'};
>+ }
>+
>+ return \%encoded_records;
> }
> 
> =head2 _marc_to_array($record)
>@@ -846,18 +994,18 @@ sub _array_to_marc {
> $record->leader($data->{leader});
> for my $field (@{$data->{fields}}) {
> my $tag = (keys %{$field})[0];
>- $field = $field->{$tag};
>+ my $field_data = $field->{$tag};
> my $marc_field;
>- if (ref($field) eq 'HASH') {
>+ if (ref($field_data) eq 'HASH') {
> my @subfields;
>- foreach my $subfield (@{$field->{subfields}}) {
>+ foreach my $subfield (@{$field_data->{subfields}}) {
> my $code = (keys %{$subfield})[0];
> push @subfields, $code;
> push @subfields, $subfield->{$code};
> }
>- $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
>+ $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
> } else {
>- $marc_field = MARC::Field->new($tag, $field)
>+ $marc_field = MARC::Field->new($tag, $field_data)
> }
> $record->append_fields($marc_field);
> }
>diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm
>index fc7eda5816..b2160cc408 100644
>--- a/Koha/SearchEngine/Elasticsearch/Search.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Search.pm
>@@ -50,7 +50,6 @@ use Koha::SearchEngine::Search;
> use Koha::Exceptions::Elasticsearch;
> use MARC::Record;
> use MARC::File::XML;
>-use MIME::Base64 qw( decode_base64 );
> use JSON;
> 
> Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
>@@ -173,7 +172,7 @@ sub search_compat {
> my $index = $offset;
> my $hits = $results->{'hits'};
> foreach my $es_record (@{$hits->{'hits'}}) {
>- $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
>+ $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
> }
> 
> # consumers of this expect a name-spaced result, we provide the default
>@@ -234,7 +233,7 @@ sub search_auth_compat {
> # it's not reproduced here yet.
> my $authtype = $rs->single;
> my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
>- my $marc = $self->decode_record_from_result($record);
>+ my $marc = $self->search_document_marc_record_decode($record);
> my $mainentry = $marc->field($auth_tag_to_report);
> my $reported_tag;
> if ($mainentry) {
>@@ -354,7 +353,7 @@ sub simple_search_compat {
> my @records;
> my $hits = $results->{'hits'};
> foreach my $es_record (@{$hits->{'hits'}}) {
>- push @records, $self->decode_record_from_result($es_record->{'_source'});
>+ push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
> }
> return (undef, \@records, $hits->{'total'});
> }
>@@ -374,31 +373,6 @@ sub extract_biblionumber {
> return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
> }
> 
>-=head2 decode_record_from_result
>- my $marc_record = $self->decode_record_from_result(@result);
>-
>-Extracts marc data from Elasticsearch result and decodes to MARC::Record object
>-
>-=cut
>-
>-sub decode_record_from_result {
>- # Result is passed in as array, will get flattened
>- # and first element will be $result
>- my ( $self, $result ) = @_;
>- if ($result->{marc_format} eq 'base64ISO2709') {
>- return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
>- }
>- elsif ($result->{marc_format} eq 'MARCXML') {
>- return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
>- }
>- elsif ($result->{marc_format} eq 'ARRAY') {
>- return $self->_array_to_marc($result->{marc_data_array});
>- }
>- else {
>- Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
>- }
>-}
>-
> =head2 max_result_window
> 
> Returns the maximum number of results that can be fetched
>diff --git a/catalogue/search.pl b/catalogue/search.pl
>index ef7f49088f..f6490a2197 100755
>--- a/catalogue/search.pl
>+++ b/catalogue/search.pl
>@@ -158,6 +158,15 @@ use Koha::Virtualshelves;
> use Koha::SearchFields;
> 
> use URI::Escape;
>+use File::Spec;
>+use File::Path qw(mkpath);
>+use Koha::Email;
>+use Koha::UploadedFiles;
>+use POSIX qw(strftime);
>+use Digest::MD5 qw(md5_hex);
>+use Encode qw(encode);
>+use YAML::XS;
>+use Carp qw(croak);
> 
> my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
> # create a new CGI object
>@@ -501,6 +510,224 @@ my $total = 0; # the total results for the whole set
> my $facets; # this object stores the faceted results that display on the left-hand of the results page
> my $results_hashref;
> 
>+my $patron = Koha::Patrons->find( $borrowernumber );
>+
>+my $export_enabled =
>+ C4::Context->preference('EnableSearchResultMARCExport') &&
>+ C4::Context->preference('SearchEngine') eq 'Elasticsearch' &&
>+ $patron && $patron->has_permission({ tools => 'export_catalog' });
>+
>+$template->param(export_enabled => $export_enabled) if $template_name eq 'catalogue/results.tt';
>+
>+if ($export_enabled) {
>+
>+ my $export = $cgi->param('export');
>+ my $preferred_format = $cgi->param('export_format');
>+
>+ my $export_custom_formats_pref = Load(C4::Context->preference('SearchResultMARCExportCustomFormats'));
>+
>+ my $custom_export_formats = {};
>+ if (ref $export_custom_formats_pref eq 'ARRAY') {
>+ for (my $i = 0; $i < @{$export_custom_formats_pref}; ++$i) {
>+ # TODO: Validate on save or trow error here instead of just ignoreing
>+ my $format = $export_custom_formats_pref->[$i];
>+ if (
>+ ref $format->{fields} eq 'ARRAY' &&
>+ @{$format->{fields}} &&
>+ $format->{name}
>+ ) {
>+ $format->{multiple} = 'ignore' unless exists $format->{multiple};
>+ $custom_export_formats->{"custom_$i"} = $format;
>+ }
>+ }
>+ }
>+ $template->param(custom_export_formats => $custom_export_formats);
>+
>+ if ($export && $preferred_format) {
>+ my $elasticsearch = $searcher->get_elasticsearch();
>+
>+ my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0;
>+ my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
>+ my $error;
>+
>+ my $results = eval {
>+ $elasticsearch->search(
>+ index => $searcher->index_name,
>+ scroll => '1m', #TODO: Syspref for scroll time limit?
>+ size => 1000, #TODO: Syspref for batch size?
>+ body => \%export_query
>+ );
>+ };
>+ if ($@) {
>+ $error = $@;
>+ $searcher->process_error($error);
>+ }
>+
>+ my @docs;
>+ for my $doc (@{$results->{hits}->{hits}}) {
>+ push @docs, $doc;
>+ }
>+
>+ my $scroll_id = $results->{_scroll_id};
>+
>+ while (@{$results->{hits}->{hits}}) {
>+ $results = $elasticsearch->scroll(
>+ scroll => '1m',
>+ scroll_id => $scroll_id
>+ );
>+ for my $doc (@{$results->{hits}->{hits}}) {
>+ push @docs, $doc;
>+ }
>+ }
>+
>+ my $message;
>+ my $export_links_html;
>+
>+ if (!$error) {
>+ my $encoded_results = {};
>+
>+ if ($preferred_format eq 'ISO2709' || $preferred_format eq 'MARCXML') {
>+ $encoded_results = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);
>+ }
>+ elsif(exists $custom_export_formats->{$preferred_format}) {
>+ my $format = $custom_export_formats->{$preferred_format};
>+ my $result;
>+
>+ my $doc_get_fields = sub {
>+ my ($doc, $fields) = @_;
>+ my @row;
>+ foreach my $field (@{$fields}) {
>+ my $values = $doc->{_source}->{$field};
>+ push @row, $values && @{$values} ? $values : '';
>+ }
>+ return \@row;
>+ };
>+
>+ my @rows = map { $doc_get_fields->($_, $format->{fields}) } @docs;
>+
>+ if($format->{multiple} eq 'ignore') {
>+ for (my $i = 0; $i < @rows; ++$i) {
>+ $rows[$i] = [map { $_->[0] } @{$rows[$i]}];
>+ }
>+ }
>+ elsif($format->{multiple} eq 'newline') {
>+ if (@{$format->{fields}} == 1) {
>+ @rows = map { [join("\n", @{$_->[0]})] } @rows;
>+ }
>+ else {
>+ croak "'newline' is only valid for single field export formats";
>+ }
>+ }
>+ elsif($format->{multiple} eq 'join') {
>+ for (my $i = 0; $i < @rows; ++$i) {
>+ $rows[$i] = [map { join("\t", @{$_}) } @{$rows[$i]}];
>+ }
>+ }
>+ else {
>+ croak "Invalid 'multiple' option: " . $format->{multiple};
>+ }
>+
>+ if (@{$format->{fields}} == 1) {
>+ @rows = map { $_->[0] } @rows;
>+ }
>+ else {
>+ # Encode CSV
>+ for (my $i = 0; $i < @rows; ++$i) {
>+ $rows[$i] = join(',', map { $_ =~ s/"/""/; "\"$_\"" } @{$rows[$i]});
>+ }
>+ }
>+ $encoded_results->{$format->{name}} = join("\n", @rows);
>+ }
>+ else {
>+ croak "Invalid export format: $preferred_format";
>+ }
>+
>+ my %format_extensions = (
>+ 'ISO2709' => '.mrc',
>+ 'MARCXML' => '.xml',
>+ );
>+
>+ my $upload_dir = Koha::UploadedFile->permanent_directory;
>+ my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";
>+ my %export_links;
>+
>+ while (my ($format, $data) = each %{$encoded_results}) {
>+ $data = encode('UTF-8', $data);
>+ my $hash = md5_hex($data);
>+ my $category = "search_marc_export";
>+ my $time = strftime "%Y%m%d_%H%M", localtime time;
>+ my $ext = exists $format_extensions{$format} ? $format_extensions{$format} : '.txt';
>+ my $filename = $category . '_' . $time . $ext;
>+ my $file_dir = File::Spec->catfile($upload_dir, $category);
>+ if ( !-d $file_dir) {
>+ mkpath $file_dir or die "Failed to create $file_dir";
>+ }
>+ my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
>+
>+ my $fh = IO::File->new($filepath, "w");
>+
>+ if ($fh) {
>+ $fh->binmode;
>+ print $fh $data;
>+ $fh->close;
>+
>+ my $size = -s $filepath;
>+ my $file = Koha::UploadedFile->new({
>+ hashvalue => $hash,
>+ filename => $filename,
>+ dir => $category,
>+ filesize => $size,
>+ owner => $borrowernumber,
>+ uploadcategorycode => 'search_marc_export',
>+ public => 0,
>+ permanent => 1,
>+ })->store;
>+ my $id = $file->_result()->get_column('id');
>+ $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";
>+ }
>+ else {
>+ croak "Failed to write \"$filepath\"";
>+ }
>+ }
>+
>+ while (my ($format, $link) = each %export_links) {
>+ $export_links_html .= "$format: <a href=\"$link\">$link</a>\n";
>+ }
>+ my $query_string = $query->{query}->{query_string}->{query};
>+ my $links_count = keys %export_links;
>+
>+ $export_links_html = $links_count > 1 ?
>+ "Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead." . $export_links_html : $export_links_html;
>+
>+ my $send_email = C4::Context->preference('SearchResultMARCExportEmail');
>+ if ($send_email) {
>+ if ($patron->email) {
>+ my $export_from_address = C4::Context->preference('SearchResultMARCExportEmailFromAddress');
>+ my $export_user_email = $patron->email;
>+ my $mail = Koha::Email->create({
>+ to => $export_user_email,
>+ from => $export_from_address,
>+ subject => "Marc export for query: $query_string",
>+ html_body => $export_links_html,
>+ });
>+ $mail->send_or_die({ transport => $patron->library->smtp_server->transport });
>+ $export_links_html .= "An email has been sent to: $export_user_email";
>+
>+ }
>+ else {
>+ $export_links_html .= "Unable to send mail, the current user has no email address set";
>+ }
>+ }
>+ $message = "The export finished successfully:" . $export_links_html;
>+ $template->param(export_message => $message);
>+ }
>+ else {
>+ $message = "An error occurred during marc export: $error",
>+ $template->param(export_error => $message);
>+ }
>+ }
>+}
>+
> eval {
> my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
> ( $error, $results_hashref, $facets ) = $searcher->search_compat(
>diff --git a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl b/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl
>new file mode 100755
>index 0000000000..9e1e855861
>--- /dev/null
>+++ b/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl
>@@ -0,0 +1,18 @@
>+use Modern::Perl;
>+
>+return {
>+ bug_number => "27859",
>+ description => "Add system preferences",
>+ up => sub {
>+ my ($args) = @_;
>+ my ($dbh, $out) = @$args{qw(dbh out)};
>+ $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });
>+ $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea') });
>+ $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo') });
>+ $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short') });
>+ $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });
>+ $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
>+ $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
>+ say $out "System preferences added";
>+ },
>+}
>diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql
>index 9a4cdbf72f..d57b4dab6a 100644
>--- a/installer/data/mysql/mandatory/sysprefs.sql
>+++ b/installer/data/mysql/mandatory/sysprefs.sql
>@@ -189,7 +189,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
> ('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'),
> ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
> ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
>-('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
>+('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
> ('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'),
> ('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
> ('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
>@@ -200,6 +200,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
> ('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
> ('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
> ('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
>+('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
> ('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
> ('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
> ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
>@@ -594,6 +595,10 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
> ('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
> ('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'),
> ('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
>+('SearchResultMARCExportCustomFormats', NULL, NULL, 'Search result MARC export custom formats', 'textarea'),
>+('SearchResultMARCExportEmail', 0, NULL, 'Send search result MARC export email', 'YesNo'),
>+('SearchResultMARCExportEmailFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short'),
>+('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
> ('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
> ('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
> ('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref
>index 2eab55ae03..57fcc96359 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref
>@@ -472,9 +472,9 @@ Administration:
> -
> - "Elasticsearch MARC format: "
> - pref: ElasticsearchMARCFormat
>- default: "ISO2709"
>+ default: "base64ISO2709"
> choices:
>- "ISO2709": "ISO2709 (exchange format)"
>+ "base64ISO2709": "ISO2709 (exchange format)"
> "ARRAY": "Searchable array"
> - ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
> - NOTE: Making the full record searchable may have a negative effect on relevance ranking of search results.
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref
>index a0d7ec430d..51ec4a62e9 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref
>@@ -131,6 +131,53 @@ Searching:
> 1: use
> 0: "don't use"
> - 'the operator "phr" in the callnumber and standard number staff interface searches.'
>+ -
>+ - pref: EnableSearchResultMARCExport
>+ type: boolean
>+ default: yes
>+ choices:
>+ yes: Enable
>+ no: Disable
>+ - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
>+ -
>+ - pref: SearchResultMARCExportCustomFormats
>+ type: textarea
>+ syntax: text/x-yaml
>+ class: code
>+ - "Define custom export formats as a YAML list of associative arrays."
>+ - "A format have the required properties \"name\", \"fields\" and an optional \"multiple\". "
>+ - "name: the human readable name of the format exposed in the staff interface."
>+ - "fields: a list of Elasticsearch fields to be included in the export."
>+ - "If fields contain a only single field the export result will contain one value per row, for multiple fields a CSV-file will be produced. "
>+ - "multiple: ignore|join|newline The behavior when handling fields with multiple values."
>+ - "ignore is the default option, only the first value will be included, the rest ignored."
>+ - "join, multiple values will be contatenated with tab as a separator."
>+ - "newline, a newline will be inserted for each value. This option does not allow \"fields\" to contain multiple fields."
>+ - "Example:"
>+ - "- name: Biblionumbers "
>+ - "&nbsp;&nbsp;fields: [local-number] "
>+ - "&nbsp;&nbsp;&nbsp;&nbsp;multiple: ignore "
>+ - "- name: Title and author "
>+ - "&nbsp;&nbsp;fields: [title, author] "
>+ - "&nbsp;&nbsp;&nbsp;&nbsp;multiple: join "
>+ -
>+ - "Limit exported MARC records from search results to a maximum of"
>+ - pref: SearchResultMARCExportLimit
>+ class: integer
>+ - "records."
>+ -
>+ - "Send an email with the export results to the current user"
>+ - pref: SearchResultMARCExportEmail
>+ type: boolean
>+ default: no
>+ choices:
>+ 1: Yes
>+ 0: No
>+ -
>+ - "Use the from-address"
>+ - pref: SearchResultMARCExportFromAddress
>+ class: short
>+ - "when mailing search results exports."
> Results display:
> -
> - pref: numSearchResultsDropdown
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt
>index 0251cbe792..1d3ba69f6a 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt
>@@ -311,6 +311,21 @@
> </div> 
> [% END %]
> 
>+ [% IF export_enabled %]
>+ <div class="btn-group">
>+ <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
>+ Export all results
>+ </button>
>+ <ul class="dropdown-menu">
>+ <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
>+ <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
>+ [% FOREACH id IN custom_export_formats.keys %]
>+ <li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page | uri %]&export=1&export_format=[% id %][% PROCESS sort_search_query %]">[% custom_export_formats.$id.name | html %]</a></li>
>+ [% END %]
>+ </ul>
>+ </div> 
>+ [% END %]
>+
> </div> 
> </div> 
> 
>@@ -339,6 +354,13 @@
> <div class="dialog alert">Error: [% query_error | html %]</div>
> [% END %]
> 
>+ [% IF ( export_message ) %]
>+ <div class="dialog message">[% export_message %]</div>
>+ [% END %]
>+ [% IF ( export_error ) %]
>+ <div class="dialog error">[% export_error %]</div>
>+ [% END %]
>+
> 
> [% IF ( total ) %]
> [% IF ( scan ) %]
>diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t
>index f29831f504..5cb808b937 100755
>--- a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t
>+++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t
>@@ -140,10 +140,10 @@ subtest 'get_elasticsearch_mappings() tests' => sub {
> 
> subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
> 
>- plan tests => 63;
>+ plan tests => 72;
> 
> t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
>- t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
>+ t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
> 
> my @mappings = (
> {
>@@ -489,7 +489,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
> ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
> is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
> 
>- my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
>+ my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
> 
> ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
> is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
>@@ -610,11 +610,63 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
> 
> is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
> 
>- $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
>+ $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
> 
> ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
> is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
> 
>+ # Search export functionality
>+ # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
>+ my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
>+
>+ for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
>+
>+ t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
>+
>+ $docs = $see->marc_records_to_documents(\@source_docs);
>+
>+ # Emulate Elasticsearch response docs structure
>+ my @es_response_docs = map { { _source => $_ } } @{$docs};
>+
>+ my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');
>+
>+ # $large_marc_record should not have been encoded as ISO2709
>+ # since exceeds maximum size, see above
>+ my @tmp = ($marc_record_1, $marc_record_2);
>+ is(
>+ $records_data->{ISO2709},
>+ join('', map { $_->as_usmarc() } @tmp),
>+ "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
>+ );
>+
>+ my $expected_marc_xml = join("\n",
>+ MARC::File::XML::header(),
>+ MARC::File::XML::record($large_marc_record, 'MARC21'),
>+ MARC::File::XML::footer()
>+ );
>+
>+ is(
>+ $records_data->{MARCXML},
>+ $expected_marc_xml,
>+ "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
>+ );
>+
>+ $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');
>+
>+ $expected_marc_xml = join("\n",
>+ MARC::File::XML::header(),
>+ join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
>+ MARC::File::XML::footer()
>+ );
>+
>+ is(
>+ $records_data->{MARCXML},
>+ $expected_marc_xml,
>+ "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
>+ );
>+
>+ }
>+
> push @mappings, {
> name => 'title',
> type => 'string',
>@@ -747,7 +799,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t
> 
> is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
> 
>- my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
>+ my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
> 
> ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
> is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
>@@ -758,7 +810,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori
> plan tests => 5;
> 
> t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
>- t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
>+ t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
> 
> my $builder = t::lib::TestBuilder->new;
> my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
>-- 
>2.35.1

Actions: View | Diff | Splinter Review

Attachments on bug 27859: 117718 | 117721 | 124892 | 124893 | 128632 | 131442 | 131443 | 132993 | 132997 | 133076 | 145243 | 145244 | 145263 | 145300 | 147927 | 147935 | 147936 | 162342 | 162343 | 163761 | 163762