From 233d3e521528f0cb3abb7fad63626a815e46bacc Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 25 Sep 2020 16:07:41 +0200 Subject: [PATCH] Bug 27859: marc search result export Enable export of staff interface search results in different marc formats. The export will be mailed to the configured mail address of the current user. This feature is Elasticsearch only. To test: 1) Apply patch 2) Run installer/data/mysql/updatedatabase.pl 3) Make sure the syspref EnableSearchResultMARCExport is enabled 4) Make sure the current user has the tools -> export_catalog permission 5) Make sure the current user has an email under your control set 6) Perform a search 7) Export the serach result by choosing a format under the "Export results" drop down 8) Verify that link(s) with exported data has been mailed to the provided emial 9) Revoke the permission in 3) and make sure exporting is no longer possible 10) Run tests in t/db_dependent/Koha/SearchEngine/Elasticsearch.t Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 214 +++++++++++++++--- Koha/SearchEngine/Elasticsearch/Search.pm | 34 +-- catalogue/search.pl | 158 +++++++++++++ ...le_search_result_marc_export_sysprefs.perl | 9 + installer/data/mysql/sysprefs.sql | 4 +- .../en/modules/admin/preferences/admin.pref | 4 +- .../modules/admin/preferences/searching.pref | 13 ++ .../prog/en/modules/catalogue/results.tt | 16 ++ .../Koha/SearchEngine/Elasticsearch.t | 64 +++++- 9 files changed, 443 insertions(+), 73 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 24ff497e27..e966fa38d5 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -41,8 +41,8 @@ use YAML::Syck; use List::Util qw( sum0 reduce ); use MARC::File::XML; -use MIME::Base64; -use Encode qw(encode); +use MIME::Base64 qw(encode_base64 decode_base64); +use Encode qw(encode decode); use Business::ISBN; use Scalar::Util qw(looks_like_number); @@ -524,7 +524,6 @@ sub marc_records_to_documents { my $control_fields_rules = $rules->{control_fields}; my $data_fields_rules = $rules->{data_fields}; my $marcflavour = lc C4::Context->preference('marcflavour'); - my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; my @record_documents; @@ -679,38 +678,187 @@ sub marc_records_to_documents { } } } + my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat'); - # TODO: Perhaps should check if $records_document non empty, but really should never be the case - $record->encoding('UTF-8'); - if ($use_array) { - $record_document->{'marc_data_array'} = $self->_marc_to_array($record); - $record_document->{'marc_format'} = 'ARRAY'; + my ($encoded_record, $format) = $self->search_document_marc_record_encode( + $record, + $preferred_format, + $marcflavour + ); + + if ($preferred_format eq 'ARRAY') { + $record_document->{'marc_data_array'} = $encoded_record; } else { - my @warnings; - { - # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) - local $SIG{__WARN__} = sub { - push @warnings, $_[0]; - }; - $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); - } - if (@warnings) { - # Suppress warnings if record length exceeded - unless (substr($record->leader(), 0, 5) eq '99999') { - foreach my $warning (@warnings) { - carp $warning; - } + $record_document->{'marc_data'} = $encoded_record; + } + $record_document->{'marc_format'} = $format; + + push @record_documents, $record_document; + } + return \@record_documents; +} + +=head2 search_document_marc_record_encode($record, $format, $marcflavour) + my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour) + +Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum +size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead. + +=over 4 + +=item C<$record> + +A MARC::Record object + +=item C<$marcflavour> + +The marcflavour to use + +=back + +=cut + +sub search_document_marc_record_encode { + my ($self, $record, $format, $marcflavour) = @_; + + $record->encoding('UTF-8'); + + if ($format eq 'ARRAY') { + return ($self->_marc_to_array($record), $format); + } + elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') { + my @warnings; + my $marc_data; + { + # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999) + local $SIG{__WARN__} = sub { + push @warnings, $_[0]; + }; + $marc_data = $record->as_usmarc(); + } + if (@warnings) { + # Suppress warnings if record length exceeded + unless (substr($record->leader(), 0, 5) eq '99999') { + foreach my $warning (@warnings) { + carp $warning; } - $record_document->{'marc_data'} = $record->as_xml_record($marcflavour); - $record_document->{'marc_format'} = 'MARCXML'; } - else { - $record_document->{'marc_format'} = 'base64ISO2709'; + return (MARC::File::XML::record($record, $marcflavour), 'MARCXML'); + } + else { + if ($format eq 'base64ISO2709') { + $marc_data = encode_base64(encode('UTF-8', $marc_data)); } + return ($marc_data, $format); } - push @record_documents, $record_document; } - return \@record_documents; + elsif ($format eq 'MARCXML') { + return (MARC::File::XML::record($record, $marcflavour), $format); + } + else { + # This should be unlikely to happen + croak "Invalid marc record serialization format: $format"; + } +} + +=head2 search_document_marc_record_decode + my $marc_record = $self->search_document_marc_record_decode(@result); + +Extract marc data from Elasticsearch result and decode to MARC::Record object + +=cut + +sub search_document_marc_record_decode { + # Result is passed in as array, will get flattened + # and first element will be $result + my ($self, $result) = @_; + if ($result->{marc_format} eq 'base64ISO2709') { + return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); + } + elsif ($result->{marc_format} eq 'MARCXML') { + return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); + } + elsif ($result->{marc_format} eq 'ARRAY') { + return $self->_array_to_marc($result->{marc_data_array}); + } + else { + Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); + } +} + +=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format) + + $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format) + +Return marc encoded records from ElasticSearch search result documents. The return value +C<$marc_records> is a hashref with encoded records keyed by MARC format. + +=over 4 + +=item C<$docs> + +An arrayref of Elasticsearch search documents + +=item C<$preferred_format> + +The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum +length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format> +is set to 'ISO2709'. + +=back + +=cut + +sub search_document_marc_records_encode_from_docs { + + my ($self, $docs, $preferred_format) = @_; + + my %encoded_records = ( + 'ISO2709' => [], + 'MARCXML' => [] + ); + + unless (exists $encoded_records{$preferred_format}) { + croak "Invalid preferred format: $preferred_format"; + } + + for my $es_record (@{$docs}) { + # Special optimized cases + my $marc_data; + my $resulting_format = $preferred_format; + if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') { + $marc_data = $es_record->{_source}{marc_data}; + } + elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') { + # Stored as UTF-8 encoded binary in index, so needs to be decoded + $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data})); + } + else { + my $record = $self->search_document_marc_record_decode($es_record->{'_source'}); + my $marcflavour = lc C4::Context->preference('marcflavour'); + ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour); + } + push @{$encoded_records{$resulting_format}}, $marc_data; + } + if (@{$encoded_records{'ISO2709'}}) { + $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}}); + } + else { + delete $encoded_records{'ISO2709'}; + } + + if (@{$encoded_records{'MARCXML'}}) { + $encoded_records{'MARCXML'} = join("\n", + MARC::File::XML::header(), + join("\n", @{$encoded_records{'MARCXML'}}), + MARC::File::XML::footer() + ); + } + else { + delete $encoded_records{'MARCXML'}; + } + + return \%encoded_records; } =head2 _marc_to_array($record) @@ -784,18 +932,18 @@ sub _array_to_marc { $record->leader($data->{leader}); for my $field (@{$data->{fields}}) { my $tag = (keys %{$field})[0]; - $field = $field->{$tag}; + my $field_data = $field->{$tag}; my $marc_field; - if (ref($field) eq 'HASH') { + if (ref($field_data) eq 'HASH') { my @subfields; - foreach my $subfield (@{$field->{subfields}}) { + foreach my $subfield (@{$field_data->{subfields}}) { my $code = (keys %{$subfield})[0]; push @subfields, $code; push @subfields, $subfield->{$code}; } - $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields); + $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields); } else { - $marc_field = MARC::Field->new($tag, $field) + $marc_field = MARC::Field->new($tag, $field_data) } $record->append_fields($marc_field); } diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index ff56ad1174..ad4b97b661 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -50,9 +50,6 @@ use Koha::SearchEngine::Search; use Koha::Exceptions::Elasticsearch; use MARC::Record; use MARC::File::XML; -use Data::Dumper; #TODO remove -use Carp qw(cluck); -use MIME::Base64; Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store )); @@ -166,7 +163,7 @@ sub search_compat { my $index = $offset; my $hits = $results->{'hits'}; foreach my $es_record (@{$hits->{'hits'}}) { - $records[$index++] = $self->decode_record_from_result($es_record->{'_source'}); + $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'}); } # consumers of this expect a name-spaced result, we provide the default @@ -227,7 +224,7 @@ sub search_auth_compat { # it's not reproduced here yet. my $authtype = $rs->single; my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : ""; - my $marc = $self->decode_record_from_result($record); + my $marc = $self->search_document_marc_record_decode($record); my $mainentry = $marc->field($auth_tag_to_report); my $reported_tag; if ($mainentry) { @@ -347,7 +344,7 @@ sub simple_search_compat { my @records; my $hits = $results->{'hits'}; foreach my $es_record (@{$hits->{'hits'}}) { - push @records, $self->decode_record_from_result($es_record->{'_source'}); + push @records, $self->search_document_marc_record_decode($es_record->{'_source'}); } return (undef, \@records, $hits->{'total'}); } @@ -367,31 +364,6 @@ sub extract_biblionumber { return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord ); } -=head2 decode_record_from_result - my $marc_record = $self->decode_record_from_result(@result); - -Extracts marc data from Elasticsearch result and decodes to MARC::Record object - -=cut - -sub decode_record_from_result { - # Result is passed in as array, will get flattened - # and first element will be $result - my ( $self, $result ) = @_; - if ($result->{marc_format} eq 'base64ISO2709') { - return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data})); - } - elsif ($result->{marc_format} eq 'MARCXML') { - return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); - } - elsif ($result->{marc_format} eq 'ARRAY') { - return $self->_array_to_marc($result->{marc_data_array}); - } - else { - Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result"); - } -} - =head2 max_result_window Returns the maximum number of results that can be fetched diff --git a/catalogue/search.pl b/catalogue/search.pl index 6876499fc8..57cd8ae5aa 100755 --- a/catalogue/search.pl +++ b/catalogue/search.pl @@ -158,6 +158,14 @@ use Koha::Virtualshelves; use Koha::SearchFields; use URI::Escape; +use Mail::Sendmail; +use File::Spec; +use File::Path qw(mkpath); +use Koha::Email; +use Koha::UploadedFiles; +use POSIX qw(strftime); +use Digest::MD5 qw(md5_hex); +use Encode qw(encode); my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold"); # create a new CGI object @@ -501,6 +509,156 @@ my $total = 0; # the total results for the whole set my $facets; # this object stores the faceted results that display on the left-hand of the results page my $results_hashref; +my $export = $cgi->param('export'); +my $preferred_format = $cgi->param('export_format'); +my $export_user_email = undef; + +if ($template_name eq 'catalogue/results.tt' && $export && $preferred_format && C4::Context->preference('SearchEngine') eq 'Elasticsearch') { + + my $uid; + my $userenv = C4::Context->userenv; + if ($userenv) { + $uid = $userenv->{number}; + if ($userenv->{emailaddress}) { + $export_user_email = $userenv->{emailaddress}; + } + else { + die "Unable to fetch user email"; + } + } + else { + die "Unable to fetch userenv"; + } + + my $patron = Koha::Patrons->find( $borrowernumber ); + + if (!($patron && $patron->has_permission({ tools => 'export_catalog' }))) { + die "Missing permission \"export_catalog\" required for exporting search results"; + } + + my $elasticsearch = $searcher->get_elasticsearch(); + + my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') || 0; + my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query}; + my $error; + + my $results = eval { + $elasticsearch->search( + index => $searcher->index_name, + scroll => '1m', #TODO: Syspref for scroll time limit? + size => 1000, #TODO: Syspref for batch size? + body => \%export_query + ); + }; + if ($@) { + $error = $@; + $searcher->process_error($error); + } + + my @docs; + for my $doc (@{$results->{hits}->{hits}}) { + push @docs, $doc; + } + + my $scroll_id = $results->{_scroll_id}; + + while (@{$results->{hits}->{hits}}) { + $results = $elasticsearch->scroll( + scroll => '1m', + scroll_id => $scroll_id + ); + for my $doc (@{$results->{hits}->{hits}}) { + push @docs, $doc; + } + } + + my $koha_email = Koha::Email->new(); + my %mail; + + if (!$error) { + my $encoded_records = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format); + + my %format_extensions = ( + 'ISO2709' => '.mrc', + 'MARCXML' => '.xml' + ); + + my $upload_dir = Koha::UploadedFile->permanent_directory; + my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha"; + my %export_links; + + while (my ($format, $data) = each %{$encoded_records}) { + $data = encode('UTF-8', $data); + my $hash = md5_hex($data); + my $category = "search_marc_export"; + my $time = strftime "%Y%m%d_%H%M", localtime time; + my $filename = $category . '_' . $time . $format_extensions{$format}; + my $file_dir = File::Spec->catfile($upload_dir, $category); + if ( !-d $file_dir) { + mkpath $file_dir or die "Failed to create $file_dir"; + } + my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}"); + + my $fh = IO::File->new($filepath, "w"); + + if ($fh) { + $fh->binmode; + print $fh $data; + $fh->close; + + my $size = -s $filepath; + my $file = Koha::UploadedFile->new({ + hashvalue => $hash, + filename => $filename, + dir => $category, + filesize => $size, + owner => $uid, + uploadcategorycode => 'search_marc_export', + public => 0, + permanent => 1, + })->store; + my $id = $file->_result()->get_column('id'); + $export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id"; + } + else { + die "Failed to write \"$filepath\""; + } + } + + if (%export_links) { + my $links_output = ''; + while (my ($format, $link) = each %export_links) { + $links_output .= "$format: $link\n"; + } + + my $query_string = $query->{query}->{query_string}->{query}; + my $links_count = keys %export_links; + my $message = $links_count > 1 ? + "Some records exceeded maximum size supported by ISO2709 and was exported as MARCXML instead.\n\n" . $links_output : $links_output; + + %mail = $koha_email->create_message_headers({ + to => $export_user_email, + from => 'noreply@ub.gu.se', + subject => "Marc export for query: $query_string", + message => $message, + }); + } + } + else { + %mail = $koha_email->create_message_headers({ + to => $export_user_email, + from => 'noreply@ub.gu.se', + subject => "Marc export error", + message => "An error occured during marc export: $error", + }); + } + $Mail::Sendmail::mailcfg{smtp} = ['smtp.gu.se']; + $Mail::Sendmail::mailcfg{port} = 25; + sendmail(%mail) || print "Error: $Mail::Sendmail::error\n"; + + $template->param(export_user_email => $export_user_email); +} + eval { my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } }; ( $error, $results_hashref, $facets ) = $searcher->search_compat( diff --git a/installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl b/installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl new file mode 100644 index 0000000000..6b9cca6905 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl @@ -0,0 +1,9 @@ +$DBversion = 'XXX'; # will be replaced by the RM +if( CheckVersion( $DBversion ) ) { + $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') }); + $dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') }); + $dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' }); + $dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' }); + + #NewVersion( $DBversion, XXXXX, "Add EnableSearchResultMARCExport system prefernce"); +} diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index 271cb3108a..58b3a93be8 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -175,7 +175,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'), ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL), ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL), -('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'), +('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'), ('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'), ('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'), ('EmailPurchaseSuggestions','0','0|EmailAddressForSuggestions|BranchEmailAddress|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'), @@ -184,6 +184,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''), ('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'), ('EnableSearchHistory','0','','Enable or disable search history','YesNo'), +('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'), ('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'), ('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'), ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'), @@ -569,6 +570,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('SCOUserJS','',NULL,'Define custom javascript for inclusion in the SCO module','free'), ('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'), ('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'), +('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'), ('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'), ('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like 192.168.1.1 192.168.0.0/24.)','Short'), ('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref index d2255fdf3f..361d609a59 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref @@ -445,9 +445,9 @@ Administration: - - "Elasticsearch MARC format: " - pref: ElasticsearchMARCFormat - default: "ISO2709" + default: "base64ISO2709" choices: - "ISO2709": "ISO2709 (exchange format)" + "base64ISO2709": "ISO2709 (exchange format)" "ARRAY": "Searchable array" -
ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable. -
NOTE: Making the full record searchable may have a negative effect on relevance ranking of search results. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index cfefd23db1..626cb2cee8 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -131,6 +131,19 @@ Searching: yes: use no: "don't use" - 'the operator "phr" in the callnumber and standard number staff client searches' + - + - pref: EnableSearchResultMARCExport + type: boolean + default: yes + choices: + yes: Enable + no: Disable + - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)." + - + - "Limit exported MARC records from search results to a maximum of" + - pref: SearchResultMARCExportLimit + class: integer + - "records." Results display: - - pref: numSearchResultsDropdown diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt index 912e3c8d35..c052dfbf2e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt @@ -301,6 +301,18 @@ [% END %] + [% IF Koha.Preference('EnableSearchResultMARCExport') && Koha.Preference('SearchEngine') == 'Elasticsearch' && CAN_user_tools_export_catalog %] +
+ + +
+ [% END %] + @@ -325,6 +337,10 @@

Error: [% query_error | html %]

[% END %] + [% IF ( export_user_email ) %] +
Export in progress, an email will results will be sent to [% export_user_email | html %]
+ [% END %] + [% IF ( total ) %] [% IF ( scan ) %] diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t index 471baf34e1..b4996178a5 100644 --- a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t +++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch.t @@ -122,10 +122,10 @@ subtest 'get_elasticsearch_mappings() tests' => sub { subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { - plan tests => 53; + plan tests => 62; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); - t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709'); my @mappings = ( { @@ -420,7 +420,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set'); is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly'); - my $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result"); is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record"); @@ -511,11 +511,63 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly'); - $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result"); is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record"); + # Search export functionality + # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs() + my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record); + + for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') { + + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format); + + $docs = $see->marc_records_to_documents(\@source_docs); + + # Emulate Elasticsearch response docs structure + my @es_response_docs = map { { _source => $_ } } @{$docs}; + + my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709'); + + # $large_marc_record should not have been encoded as ISO2709 + # since exceeds maximum size, see above + my @tmp = ($marc_record_1, $marc_record_2); + is( + $records_data->{ISO2709}, + join('', map { $_->as_usmarc() } @tmp), + "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\"" + ); + + my $expected_marc_xml = join("\n", + MARC::File::XML::header(), + MARC::File::XML::record($large_marc_record, 'MARC21'), + MARC::File::XML::footer() + ); + + is( + $records_data->{MARCXML}, + $expected_marc_xml, + "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\"" + ); + + $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML'); + + $expected_marc_xml = join("\n", + MARC::File::XML::header(), + join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs), + MARC::File::XML::footer() + ); + + is( + $records_data->{MARCXML}, + $expected_marc_xml, + "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\"" + ); + + } + push @mappings, { name => 'title', type => 'string', @@ -633,7 +685,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly'); - my $decoded_marc_record = $see->decode_record_from_result($docs->[0]); + my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result"); is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record"); @@ -644,7 +696,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori plan tests => 2; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); - t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); + t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709'); my $builder = t::lib::TestBuilder->new; my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{ -- 2.20.1