From 8e7ecc1b828d70df38ca862ff14c5aa0f8075eb1 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 25 Sep 2020 16:07:41 +0200 Subject: [PATCH] Bug 27859: marc search result export Enable export of staff interface search results in different marc formats. The export will be mailed to the configured mail address of the current user. This feature is Elasticsearch only. To test: 1) Apply patch 2) Run installer/data/mysql/updatedatabase.pl 3) Make sure the syspref EnableSearchResultMARCExport is enabled 4) Make sure the current user has the tools -> export_catalog permission 5) Make sure the current user has an email under your control set 6) Perform a search 7) Export the serach result by choosing a format under the "Export results" drop down 8) Verify that link(s) with exported data has been mailed to the provided emial 9) Revoke the permission in 3) and make sure exporting is no longer possible 10) Run tests in t/db_dependent/Koha/SearchEngine/Elasticsearch.t Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 214 +++++++++++++++--- Koha/SearchEngine/Elasticsearch/Search.pm | 32 +-- catalogue/search.pl | 154 +++++++++++++ ...able_search_result_marc_export_sysprefs.pl | 16 ++ installer/data/mysql/mandatory/sysprefs.sql | 5 +- .../en/modules/admin/preferences/admin.pref | 4 +- .../modules/admin/preferences/searching.pref | 18 ++ .../prog/en/modules/catalogue/results.tt | 16 ++ .../Koha/SearchEngine/Elasticsearch.t | 64 +++++- 9 files changed, 452 insertions(+), 71 deletions(-) create mode 100755 installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index f4a2f82713..75d46ea453 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -41,8 +41,8 @@ use YAML::XS; use List::Util qw( sum0 ); use MARC::File::XML; -use MIME::Base64 qw( encode_base64 ); -use Encode qw( encode ); +use MIME::Base64 qw(encode_base64 decode_base64); +use Encode qw(encode decode); use Business::ISBN; use Scalar::Util qw( looks_like_number ); @@ -541,7 +541,6 @@ sub marc_records_to_documents { my $control_fields_rules = $rules->{control_fields}; my $data_fields_rules = $rules->{data_fields}; my $marcflavour = lc C4::Context->preference('marcflavour'); - my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; my @record_documents; @@ -741,38 +740,187 @@ sub marc_records_to_documents { } } } + my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); - # TODO: Perhaps should check if $records_document non empty, but really should never be the case - $record->encoding('UTF-8'); - if ($use_array) { - $record_document->{'marc_data_array'} = $self->_marc_to_array($record); - $record_document->{'marc_format'} = 'ARRAY'; + my ($encoded_record, $format) = $self->search_document_marc_record_encode( + $record, + $preferred_format, + $marcflavour + ); + + if ($preferred_format eq 'ARRAY') { + $record_document->{'marc_data_array'} = $encoded_record; } else { - my @warnings; - { - # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) - local $SIG{__WARN__} = sub { - push @warnings, $_[0]; - }; - $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); - } - if (@warnings) { - # Suppress warnings if record length exceeded - unless (substr($record->leader(), 0, 5) eq '99999') { - foreach my $warning (@warnings) { - carp $warning; - } + $record_document->{'marc_data'} = $encoded_record; + } + $record_document->{'marc_format'} = $format; + + push @record_documents, $record_document; + } + return \@record_documents; +} + +=head2 search_document_marc_record_encode($record, $format, $marcflavour) + my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) + +Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum +size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. + +=over 4 + +=item C<$record> + +A MARC::Record object + +=item C<$marcflavour> + +The marcflavour to use + +=back + +=cut + +sub search_document_marc_record_encode { + my ($self, $record, $format, $marcflavour) = @_; + + $record->encoding('UTF-8'); + + if ($format eq 'ARRAY') { + return ($self->_marc_to_array($record), $format); + } + elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { + my @warnings; + my $marc_data; + { + # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) + local $SIG{__WARN__} = sub { + push @warnings, $_[0]; + }; + $marc_data = $record->as_usmarc(); + } + if (@warnings) { + # Suppress warnings if record length exceeded + unless (substr($record->leader(), 0, 5) eq '99999') { + foreach my $warning (@warnings) { + carp $warning; } - $record_document->{'marc_data'} = $record->as_xml_record($marcflavour); - $record_document->{'marc_format'} = 'MARCXML'; } - else { - $record_document->{'marc_format'} = 'base64ISO2709'; + return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); + } + else { + if ($format eq 'base64ISO2709') { + $marc_data = encode_base64(encode('UTF-8', $marc_data)); } + return ($marc_data, $format); } - push @record_documents, $record_document; } - return \@record_documents; + elsif ($format eq 'MARCXML') { + return (MARC::File::XML::record($record, $marcflavour), $format); + } + else { + # This should be unlikely to happen + croak "Invalid marc record serialization format: $format"; + } +} + +=head2 search_document_marc_record_decode + my $marc_record = $self->search_document_marc_record_decode(@result); + +Extract marc data from Elasticsearch result and decode to MARC::Record object + +=cut + +sub search_document_marc_record_decode { + # Result is passed in as array, will get flattened + # and first element will be $result + my ($self, $result) = @_; + if ($result->{marc_format} eq 'base64ISO2709') { + return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); + } + elsif ($result->{marc_format} eq 'MARCXML') { + return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); + } + elsif ($result->{marc_format} eq 'ARRAY') { + return $self->_array_to_marc($result->{marc_data_array}); + } + else { + Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); + } +} + +=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) + + $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) + +Return marc encoded records from ElasticSearch search result documents. The return value +C<$marc_records> is a hashref with encoded records keyed by MARC format. + +=over 4 + +=item C<$docs> + +An arrayref of Elasticsearch search documents + +=item C<$preferred_format> + +The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum +length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> +is set to 'ISO2709'. + +=back + +=cut + +sub search_document_marc_records_encode_from_docs { + + my ($self, $docs, $preferred_format) = @_; + + my %encoded_records = ( + 'ISO2709' => [], + 'MARCXML' => [] + ); + + unless (exists $encoded_records{$preferred_format}) { + croak "Invalid preferred format: $preferred_format"; + } + + for my $es_record (@{$docs}) { + # Special optimized cases + my $marc_data; + my $resulting_format = $preferred_format; + if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { + $marc_data = $es_record->{_source}{marc_data}; + } + elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { + # Stored as UTF-8 encoded binary in index, so needs to be decoded + $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); + } + else { + my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); + my $marcflavour = lc C4::Context->preference('marcflavour'); + ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); + } + push @{$encoded_records{$resulting_format}}, $marc_data; + } + if (@{$encoded_records{'ISO2709'}}) { + $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); + } + else { + delete $encoded_records{'ISO2709'}; + } + + if (@{$encoded_records{'MARCXML'}}) { + $encoded_records{'MARCXML'} = join("\n", + MARC::File::XML::header(), + join("\n", @{$encoded_records{'MARCXML'}}), + MARC::File::XML::footer() + ); + } + else { + delete $encoded_records{'MARCXML'}; + } + + return \%encoded_records; } =head2 _marc_to_array($record) @@ -846,18 +994,18 @@ sub _array_to_marc { $record->leader($data->{leader}); for my $field (@{$data->{fields}}) { my $tag = (keys %{$field})[0]; - $field = $field->{$tag}; + my $field_data = $field->{$tag}; my $marc_field; - if (ref($field) eq 'HASH') { + if (ref($field_data) eq 'HASH') { my @subfields; - foreach my $subfield (@{$field->{subfields}}) { + foreach my $subfield (@{$field_data->{subfields}}) { my $code = (keys %{$subfield})[0]; push @subfields, $code; push @subfields, $subfield->{$code}; } - $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); + $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); } else { - $marc_field = MARC::Field->new($tag, $field) + $marc_field = MARC::Field->new($tag, $field_data) } $record->append_fields($marc_field); } diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index e753220948..ed359adb9f 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -50,7 +50,6 @@ use Koha::SearchEngine::Search; use Koha::Exceptions::Elasticsearch; use MARC::Record; use MARC::File::XML; -use MIME::Base64 qw( decode_base64 ); Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store )); @@ -164,7 +163,7 @@ sub search_compat { my $index = $offset; my $hits = $results->{'hits'}; foreach my $es_record (@{$hits->{'hits'}}) { - $records[$index++] = $self->decode_record_from_result($es_record->{'_source'}); + $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'}); } # consumers of this expect a name-spaced result, we provide the default @@ -225,7 +224,7 @@ sub search_auth_compat { # it's not reproduced here yet. my $authtype = $rs->single; my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : ""; - my $marc = $self->decode_record_from_result($record); + my $marc = $self->search_document_marc_record_decode($record); my $mainentry = $marc->field($auth_tag_to_report); my $reported_tag; if ($mainentry) { @@ -345,7 +344,7 @@ sub simple_search_compat { my @records; my $hits = $results->{'hits'}; foreach my $es_record (@{$hits->{'hits'}}) { - push @records, $self->decode_record_from_result($es_record->{'_source'}); + push @records, $self->search_document_marc_record_decode($es_record->{'_source'}); } return (undef, \@records, $hits->{'total'}); } @@ -365,31 +364,6 @@ sub extract_biblionumber { return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord ); } -=head2 decode_record_from_result - my $marc_record = $self->decode_record_from_result(@result); - -Extracts marc data from Elasticsearch result and decodes to MARC::Record object - -=cut - -sub decode_record_from_result { - # Result is passed in as array, will get flattened - # and first element will be $result - my ( $self, $result ) = @_; - if ($result->{marc_format} eq 'base64ISO2709') { - return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); - } - elsif ($result->{marc_format} eq 'MARCXML') { - return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); - } - elsif ($result->{marc_format} eq 'ARRAY') { - return $self->_array_to_marc($result->{marc_data_array}); - } - else { - Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); - } -} - =head2 max_result_window Returns the maximum number of results that can be fetched diff --git a/catalogue/search.pl b/catalogue/search.pl index 5da9c063f3..16a1d3e54a 100755 --- a/catalogue/search.pl +++ b/catalogue/search.pl @@ -158,6 +158,14 @@ use Koha::Virtualshelves; use Koha::SearchFields; use URI::Escape; +use Mail::Sendmail; +use File::Spec; +use File::Path qw(mkpath); +use Koha::Email; +use Koha::UploadedFiles; +use POSIX qw(strftime); +use Digest::MD5 qw(md5_hex); +use Encode qw(encode); my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold"); # create a new CGI object @@ -497,6 +505,152 @@ my $total = 0; # the total results for the whole set my $facets; # this object stores the faceted results that display on the left-hand of the results page my $results_hashref; +my $export = $cgi->param('export'); +my $preferred_format = $cgi->param('export_format'); +my $export_user_email = undef; + +if ($template_name eq 'catalogue/results.tt' && $export && $preferred_format && C4::Context->preference('SearchEngine') eq 'Elasticsearch') { + + my $patron = Koha::Patrons->find( $borrowernumber ); + + if ($patron) { + if ($patron->email) { + $export_user_email = $patron->email; + } + else { + die "Unable to fetch user email"; + } + } + else { + die "Unable to fetch user"; + } + + if (!($patron && $patron->has_permission({ tools => 'export_catalog' }))) { + die "Missing permission \"export_catalog\" required for exporting search results"; + } + + my $elasticsearch = $searcher->get_elasticsearch(); + + my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0; + my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query}; + my $error; + + my $results = eval { + $elasticsearch->search( + index => $searcher->index_name, + scroll => '1m', #TODO: Syspref for scroll time limit? + size => 1000, #TODO: Syspref for batch size? + body => \%export_query + ); + }; + if ($@) { + $error = $@; + $searcher->process_error($error); + } + + my @docs; + for my $doc (@{$results->{hits}->{hits}}) { + push @docs, $doc; + } + + my $scroll_id = $results->{_scroll_id}; + + while (@{$results->{hits}->{hits}}) { + $results = $elasticsearch->scroll( + scroll => '1m', + scroll_id => $scroll_id + ); + for my $doc (@{$results->{hits}->{hits}}) { + push @docs, $doc; + } + } + + my $koha_email = Koha::Email->new(); + my %mail; + my $export_from_address = C4::Context->preference('SearchResultMARCExportFromAddress'); + + if (!$error) { + my $encoded_records = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format); + + my %format_extensions = ( + 'ISO2709' => '.mrc', + 'MARCXML' => '.xml' + ); + + my $upload_dir = Koha::UploadedFile->permanent_directory; + my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha"; + my %export_links; + + while (my ($format, $data) = each %{$encoded_records}) { + $data = encode('UTF-8', $data); + my $hash = md5_hex($data); + my $category = "search_marc_export"; + my $time = strftime "%Y%m%d_%H%M", localtime time; + my $filename = $category . '_' . $time . $format_extensions{$format}; + my $file_dir = File::Spec->catfile($upload_dir, $category); + if ( !-d $file_dir) { + mkpath $file_dir or die "Failed to create $file_dir"; + } + my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}"); + + my $fh = IO::File->new($filepath, "w"); + + if ($fh) { + $fh->binmode; + print $fh $data; + $fh->close; + + my $size = -s $filepath; + my $file = Koha::UploadedFile->new({ + hashvalue => $hash, + filename => $filename, + dir => $category, + filesize => $size, + owner => $borrowernumber, + uploadcategorycode => 'search_marc_export', + public => 0, + permanent => 1, + })->store; + my $id = $file->_result()->get_column('id'); + $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id"; + } + else { + die "Failed to write \"$filepath\""; + } + } + + if (%export_links) { + my $links_output = ''; + while (my ($format, $link) = each %export_links) { + $links_output .= "$format: $link\n"; + } + + my $query_string = $query->{query}->{query_string}->{query}; + my $links_count = keys %export_links; + my $message = $links_count > 1 ? + "Some records exceeded the maximum size supported by ISO2709 and was exported as MARCXML instead.\n\n" . $links_output : $links_output; + + %mail = $koha_email->create_message_headers({ + to => $export_user_email, + from => $export_from_address, + subject => "Marc export for query: $query_string", + message => $message, + }); + } + } + else { + %mail = $koha_email->create_message_headers({ + to => $export_user_email, + from => $export_from_address, + subject => "Marc export error", + message => "An error occurred during marc export: $error", + }); + } + sendmail(%mail) || print "Error: $Mail::Sendmail::error\n"; + + $template->param(export_user_email => $export_user_email); +} + eval { my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } }; ( $error, $results_hashref, $facets ) = $searcher->search_compat( diff --git a/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl b/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl new file mode 100755 index 0000000000..847eac921a --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_27859-add_enable_search_result_marc_export_sysprefs.pl @@ -0,0 +1,16 @@ +use Modern::Perl; + +return { + bug_number => "27859", + description => "Add system preferences", + up => sub { + my ($args) = @_; + my ($dbh, $out) = @$args{qw(dbh out)}; + $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') }); + $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') }); + $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short') }); + $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' }); + $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' }); + say $out "System preferences added"; + }, +} diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index 1384649444..e13c486070 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -189,7 +189,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('EdifactInvoiceImport', 'automatic', 'automatic|manual', "If on, don't auto-import EDI invoices, just keep them in the database with the status 'new'", 'Choice'), ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL), ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL), -('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'), +('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'), ('ElasticsearchCrossFields', '1', '', 'Enable "cross_fields" option for searches using Elastic search.', 'YesNo'), ('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'), ('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'), @@ -199,6 +199,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''), ('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'), ('EnableSearchHistory','0','','Enable or disable search history','YesNo'), +('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'), ('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'), ('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'), ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'), @@ -594,6 +595,8 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'), ('SearchLimitLibrary', 'homebranch', 'homebranch|holdingbranch|both', "When limiting search results with a library or library group, use the item's home library, or holding library, or both.", 'Choice'), ('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'), +('SearchResultMARCExportFromAddress', NULL, NULL, 'Search result MARC export email from-address', 'short'), +('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'), ('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'), ('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like 192.168.1.1 192.168.0.0/24.)','Short'), ('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref index 3263d6d031..333517654f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref @@ -464,9 +464,9 @@ Administration: - - "Elasticsearch MARC format: " - pref: ElasticsearchMARCFormat - default: "ISO2709" + default: "base64ISO2709" choices: - "ISO2709": "ISO2709 (exchange format)" + "base64ISO2709": "ISO2709 (exchange format)" "ARRAY": "Searchable array" -
ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable. -
NOTE: Making the full record searchable may have a negative effect on relevance ranking of search results. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index a0d7ec430d..2306422c30 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -131,6 +131,24 @@ Searching: 1: use 0: "don't use" - 'the operator "phr" in the callnumber and standard number staff interface searches.' + - + - pref: EnableSearchResultMARCExport + type: boolean + default: yes + choices: + yes: Enable + no: Disable + - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)." + - + - "Limit exported MARC records from search results to a maximum of" + - pref: SearchResultMARCExportLimit + class: integer + - "records." + - + - "Use the from-address" + - pref: SearchResultMARCExportFromAddress + class: short + - "when mailing marc exports of search results." Results display: - - pref: numSearchResultsDropdown diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt index 4115ddacf7..3744d7d8a7 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt @@ -310,6 +310,18 @@ [% END %] + [% IF Koha.Preference('EnableSearchResultMARCExport') && Koha.Preference('SearchEngine') == 'Elasticsearch' && CAN_user_tools_export_catalog %] +
+ + +
+ [% END %] + @@ -337,6 +349,10 @@

Error: [% query_error | html %]

[% END %] + [% IF ( export_user_email ) %] +
Export in progress, an email will results will be sent to [% export_user_email | html %]
+ [% END %] + [% IF ( total ) %] [% IF ( scan ) %] diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t index 78d0c4e331..bbab9d8596 100755 --- a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t +++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t @@ -139,10 +139,10 @@ subtest 'get_elasticsearch_mappings() tests' => sub { subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { - plan tests => 63; + plan tests => 72; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); - t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709'); my @mappings = ( { @@ -488,7 +488,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set'); is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly'); - my $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result"); is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record"); @@ -609,11 +609,63 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly'); - $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result"); is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record"); + # Search export functionality + # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs() + my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record); + + for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') { + + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format); + + $docs = $see->marc_records_to_documents(\@source_docs); + + # Emulate Elasticsearch response docs structure + my @es_response_docs = map { { _source => $_ } } @{$docs}; + + my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709'); + + # $large_marc_record should not have been encoded as ISO2709 + # since exceeds maximum size, see above + my @tmp = ($marc_record_1, $marc_record_2); + is( + $records_data->{ISO2709}, + join('', map { $_->as_usmarc() } @tmp), + "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\"" + ); + + my $expected_marc_xml = join("\n", + MARC::File::XML::header(), + MARC::File::XML::record($large_marc_record, 'MARC21'), + MARC::File::XML::footer() + ); + + is( + $records_data->{MARCXML}, + $expected_marc_xml, + "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\"" + ); + + $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML'); + + $expected_marc_xml = join("\n", + MARC::File::XML::header(), + join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs), + MARC::File::XML::footer() + ); + + is( + $records_data->{MARCXML}, + $expected_marc_xml, + "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\"" + ); + + } + push @mappings, { name => 'title', type => 'string', @@ -746,7 +798,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly'); - my $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result"); is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record"); @@ -757,7 +809,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori plan tests => 5; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); - t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709'); my $builder = t::lib::TestBuilder->new; my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{ -- 2.34.0