Attachment #117721 for bug #27859

View | Details | Raw Unified | Return to bug 27859
Collapse All | Expand All





use List::Util qw( sum0 reduce );
use MARC::File::XML;
use MIME::Base64 qw(encode_base64 decode_base64);
use Encode qw(encode decode);
use Business::ISBN;
use Scalar::Util qw(looks_like_number);


    my $control_fields_rules = $rules->{control_fields};
    my $data_fields_rules = $rules->{data_fields};
    my $marcflavour = lc C4::Context->preference('marcflavour');
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';

    my @record_documents;


                }
            }
        }
        my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');

        my ($encoded_record, $format) = $self->search_document_marc_record_encode(
            $record,
            $preferred_format,
            $marcflavour
        );

        if ($preferred_format eq 'ARRAY') {
            $record_document->{'marc_data_array'} = $encoded_record;
        } else {
            $record_document->{'marc_data'} = $encoded_record;
        }
        $record_document->{'marc_format'} = $format;

        push @record_documents, $record_document;
    }
    return \@record_documents;
}

=head2 search_document_marc_record_encode($record, $format, $marcflavour)
    my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)

Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.

=over 4

=item C<$record>

A MARC::Record object

=item C<$marcflavour>

The marcflavour to use

=back

=cut

sub search_document_marc_record_encode {
    my ($self, $record, $format, $marcflavour) = @_;

    $record->encoding('UTF-8');

    if ($format eq 'ARRAY') {
        return ($self->_marc_to_array($record), $format);
    }
    elsif ($format eq 'base64ISO2709' || $format eq 'ISO2709') {
        my @warnings;
        my $marc_data;
        {
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
            local $SIG{__WARN__} = sub {
                push @warnings, $_[0];
            };
            $marc_data = $record->as_usmarc();
        }
        if (@warnings) {
            # Suppress warnings if record length exceeded
            unless (substr($record->leader(), 0, 5) eq '99999') {
                foreach my $warning (@warnings) {
                    carp $warning;
                }
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
                $record_document->{'marc_format'} = 'MARCXML';
            }
            return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
        }
        else {
            if ($format eq 'base64ISO2709') {
                $marc_data = encode_base64(encode('UTF-8', $marc_data));
            }
            return ($marc_data, $format);
        }
        push @record_documents, $record_document;
    }
    elsif ($format eq 'MARCXML') {
        return (MARC::File::XML::record($record, $marcflavour), $format);
    }
    else {
        # This should be unlikely to happen
        croak "Invalid marc record serialization format: $format";
    }
}

=head2 search_document_marc_record_decode
    my $marc_record = $self->search_document_marc_record_decode(@result);

Extract marc data from Elasticsearch result and decode to MARC::Record object

=cut

sub search_document_marc_record_decode {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ($self, $result) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)

    $records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)

Return marc encoded records from ElasticSearch search result documents. The return value
C<$marc_records> is a hashref with encoded records keyed by MARC format.

=over 4

=item C<$docs>

An arrayref of Elasticsearch search documents

=item C<$preferred_format>

The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
is set to 'ISO2709'.

=back

=cut

sub search_document_marc_records_encode_from_docs {

    my ($self, $docs, $preferred_format) = @_;

    my %encoded_records = (
        'ISO2709' => [],
        'MARCXML' => []
    );

    unless (exists $encoded_records{$preferred_format}) {
       croak "Invalid preferred format: $preferred_format";
    }

    for my $es_record (@{$docs}) {
        # Special optimized cases
        my $marc_data;
        my $resulting_format = $preferred_format;
        if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
            $marc_data = $es_record->{_source}{marc_data};
        }
        elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
            # Stored as UTF-8 encoded binary in index, so needs to be decoded
            $marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
        }
        else {
            my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
            my $marcflavour = lc C4::Context->preference('marcflavour');
            ($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
        }
        push @{$encoded_records{$resulting_format}}, $marc_data;
    }
    if (@{$encoded_records{'ISO2709'}}) {
        $encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
    }
    else {
        delete $encoded_records{'ISO2709'};
    }

    if (@{$encoded_records{'MARCXML'}}) {
        $encoded_records{'MARCXML'} = join("\n",
            MARC::File::XML::header(),
            join("\n", @{$encoded_records{'MARCXML'}}),
            MARC::File::XML::footer()
        );
    }
    else {
        delete $encoded_records{'MARCXML'};
    }

    return \%encoded_records;
}

=head2 _marc_to_array($record)

    $record->leader($data->{leader});
    for my $field (@{$data->{fields}}) {
        my $tag = (keys %{$field})[0];
        my $field_data = $field->{$tag};
        my $marc_field;
        if (ref($field_data) eq 'HASH') {
            my @subfields;
            foreach my $subfield (@{$field_data->{subfields}}) {
                my $code = (keys %{$subfield})[0];
                push @subfields, $code;
                push @subfields, $subfield->{$code};
            }
            $marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
        } else {
            $marc_field = MARC::Field->new($tag, $field_data)
        }
        $record->append_fields($marc_field);
    }




use Koha::Exceptions::Elasticsearch;
use MARC::Record;
use MARC::File::XML;
use Data::Dumper; #TODO remove
use Carp qw(cluck);
use MIME::Base64;

Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));


    my $index = $offset;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        $records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
    }

    # consumers of this expect a name-spaced result, we provide the default

            # it's not reproduced here yet.
            my $authtype           = $rs->single;
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
            my $marc               = $self->search_document_marc_record_decode($record);
            my $mainentry          = $marc->field($auth_tag_to_report);
            my $reported_tag;
            if ($mainentry) {

    my @records;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
    }
    return (undef, \@records, $hits->{'total'});
}

    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
}

=head2 decode_record_from_result
    my $marc_record = $self->decode_record_from_result(@result);

Extracts marc data from Elasticsearch result and decodes to MARC::Record object

=cut

sub decode_record_from_result {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ( $self, $result ) = @_;
    if ($result->{marc_format} eq 'base64ISO2709') {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    }
    elsif ($result->{marc_format} eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    elsif ($result->{marc_format} eq 'ARRAY') {
        return $self->_array_to_marc($result->{marc_data_array});
    }
    else {
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
    }
}

=head2 max_result_window

Returns the maximum number of results that can be fetched

Lines 158-163 use Koha::Virtualshelves; Link Here

(-)a/catalogue/search.pl (+158 lines)
158	use Koha::SearchFields;	158	use Koha::SearchFields;
159		159
160	use URI::Escape;	160	use URI::Escape;
		161	use Mail::Sendmail;
		162	use File::Spec;
		163	use File::Path qw(mkpath);
		164	use Koha::Email;
		165	use Koha::UploadedFiles;
		166	use POSIX qw(strftime);
		167	use Digest::MD5 qw(md5_hex);
		168	use Encode qw(encode);
161		169
162	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");	170	my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold");
163	# create a new CGI object	171	# create a new CGI object
Lines 501-506 my $total = 0; # the total results for the whole set Link Here
501	my $facets; # this object stores the faceted results that display on the left-hand of the results page	509	my $facets; # this object stores the faceted results that display on the left-hand of the results page
502	my $results_hashref;	510	my $results_hashref;
503		511
		512	my $export = $cgi->param('export');
		513	my $preferred_format = $cgi->param('export_format');
		514	my $export_user_email = undef;
		515
		516	if ($template_name eq 'catalogue/results.tt' && $export && $preferred_format && C4::Context->preference('SearchEngine') eq 'Elasticsearch') {
		517
		518	my $uid;
		519	my $userenv = C4::Context->userenv;
		520	if ($userenv) {
		521	$uid = $userenv->{number};
		522	if ($userenv->{emailaddress}) {
		523	$export_user_email = $userenv->{emailaddress};
		524	}
		525	else {
		526	die "Unable to fetch user email";
		527	}
		528	}
		529	else {
		530	die "Unable to fetch userenv";
		531	}
		532
		533	my $patron = Koha::Patrons->find( $borrowernumber );
		534
		535	if (!($patron && $patron->has_permission({ tools => 'export_catalog' }))) {
		536	die "Missing permission \"export_catalog\" required for exporting search results";
		537	}
		538
		539	my $elasticsearch = $searcher->get_elasticsearch();
		540
		541	my $size_limit = C4::Context->preference('SearchResultMARCExportLimit') \|\| 0;
		542	my %export_query = $size_limit ? (%{$query}, (size => $size_limit)) : %{$query};
		543	my $error;
		544
		545	my $results = eval {
		546	$elasticsearch->search(
		547	index => $searcher->index_name,
		548	scroll => '1m', #TODO: Syspref for scroll time limit?
		549	size => 1000, #TODO: Syspref for batch size?
		550	body => \%export_query
		551	);
		552	};
		553	if ($@) {
		554	$error = $@;
		555	$searcher->process_error($error);
		556	}
		557
		558	my @docs;
		559	for my $doc (@{$results->{hits}->{hits}}) {
		560	push @docs, $doc;
		561	}
		562
		563	my $scroll_id = $results->{_scroll_id};
		564
		565	while (@{$results->{hits}->{hits}}) {
		566	$results = $elasticsearch->scroll(
		567	scroll => '1m',
		568	scroll_id => $scroll_id
		569	);
		570	for my $doc (@{$results->{hits}->{hits}}) {
		571	push @docs, $doc;
		572	}
		573	}
		574
		575	my $koha_email = Koha::Email->new();
576	my %mail;
577
578	if (!$error) {
579	my $encoded_records = $searcher->search_document_marc_records_encode_from_docs(\@docs, $preferred_format);
580
581	my %format_extensions = (
582	'ISO2709' => '.mrc',
583	'MARCXML' => '.xml'
584	);
585
586	my $upload_dir = Koha::UploadedFile->permanent_directory;
587	my $base_url = C4::Context->preference("staffClientBaseURL") . "/cgi-bin/koha";
588	my %export_links;
589
590	while (my ($format, $data) = each %{$encoded_records}) {
591	$data = encode('UTF-8', $data);
592	my $hash = md5_hex($data);
593	my $category = "search_marc_export";
594	my $time = strftime "%Y%m%d_%H%M", localtime time;
595	my $filename = $category . '_' . $time . $format_extensions{$format};
596	my $file_dir = File::Spec->catfile($upload_dir, $category);
597	if ( !-d $file_dir) {
598	mkpath $file_dir or die "Failed to create $file_dir";
599	}
600	my $filepath = File::Spec->catfile($file_dir, "${hash}_${filename}");
601
602	my $fh = IO::File->new($filepath, "w");
603
604	if ($fh) {
605	$fh->binmode;
606	print $fh $data;
607	$fh->close;
608
609	my $size = -s $filepath;
610	my $file = Koha::UploadedFile->new({
611	hashvalue => $hash,
612	filename => $filename,
613	dir => $category,
614	filesize => $size,
615	owner => $uid,
616	uploadcategorycode => 'search_marc_export',
617	public => 0,
618	permanent => 1,
619	})->store;
620	my $id = $file->_result()->get_column('id');
621	$export_links{$format} = "$base_url/tools/upload.pl?op=download&id=$id";
622	}
623	else {
624	die "Failed to write \"$filepath\"";
625	}
626	}
627
628	if (%export_links) {
629	my $links_output = '';
630	while (my ($format, $link) = each %export_links) {
631	$links_output .= "$format: $link\n";
632	}
633
634	my $query_string = $query->{query}->{query_string}->{query};
635	my $links_count = keys %export_links;
636	my $message = $links_count > 1 ?
637	"Some records exceeded maximum size supported by ISO2709 and was exported as MARCXML instead.\n\n" . $links_output : $links_output;
638
639	%mail = $koha_email->create_message_headers({
640	to => $export_user_email,
641	from => 'noreply@ub.gu.se',
642	subject => "Marc export for query: $query_string",
643	message => $message,
644	});
645	}
646	}
647	else {
648	%mail = $koha_email->create_message_headers({
649	to => $export_user_email,
650	from => 'noreply@ub.gu.se',
651	subject => "Marc export error",
652	message => "An error occured during marc export: $error",
653	});
654	}
655	$Mail::Sendmail::mailcfg{smtp} = ['smtp.gu.se'];
656	$Mail::Sendmail::mailcfg{port} = 25;
657	sendmail(%mail) \|\| print "Error: $Mail::Sendmail::error\n";
658
659	$template->param(export_user_email => $export_user_email);
660	}
661
504	eval {	662	eval {
505	my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };	663	my $itemtypes = { map { $_->{itemtype} => $_ } @{ Koha::ItemTypes->search_with_localization->unblessed } };
506	( $error, $results_hashref, $facets ) = $searcher->search_compat(	664	( $error, $results_hashref, $facets ) = $searcher->search_compat(

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_xxxxx-add_enable_search_result_marc_export_sysprefs.perl (+9 lines)
	1	$DBversion = 'XXX'; # will be replaced by the RM
	2	if( CheckVersion( $DBversion ) ) {
	3	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('EnableSearchResultMARCExport', 1, NULL, 'Enable search result MARC export', 'YesNo') });
	4	$dbh->do(q{ INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES ('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer') });
	5	$dbh->do(q{ UPDATE systempreferences SET options = 'base64ISO2709\|ARRAY' WHERE variable = 'ElasticsearchMARCFormat' });
	6	$dbh->do(q{ UPDATE systempreferences SET value = 'base64ISO2709' WHERE variable = 'ElasticsearchMARCFormat' AND value = 'ISO2709' });
	7
	8	#NewVersion( $DBversion, XXXXX, "Add EnableSearchResultMARCExport system prefernce");
	9	}




('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
('EmailPurchaseSuggestions','0','0|EmailAddressForSuggestions|BranchEmailAddress|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'),

('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),

('SCOUserJS','',NULL,'Define custom javascript for inclusion in the SCO module','free'),
('SearchEngine','Zebra','Elasticsearch|Zebra','Search Engine','Choice'),
('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
('SelfCheckHelpMessage','','70|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),

Lines 445-453 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-2 / +2 lines)
445	-	445	-
446	- "Elasticsearch MARC format: "	446	- "Elasticsearch MARC format: "
447	- pref: ElasticsearchMARCFormat	447	- pref: ElasticsearchMARCFormat
448	default: "ISO2709"	448	default: "base64ISO2709"
449	choices:	449	choices:
450	"ISO2709": "ISO2709 (exchange format)"	450	"base64ISO2709": "ISO2709 (exchange format)"
451	"ARRAY": "Searchable array"	451	"ARRAY": "Searchable array"
452	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.	452	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
453	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.	453	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.




                  yes: use
                  no: "don't use"
            - 'the operator "phr" in the callnumber and standard number staff client searches'
        -
            - pref: EnableSearchResultMARCExport
              type: boolean
              default: yes
              choices:
                  yes: Enable
                  no: Disable
            - "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
        -
            - "Limit exported MARC records from search results to a maximum of"
            - pref: SearchResultMARCExportLimit
              class: integer
            - "records."
    Results display:
        -
            - pref: numSearchResultsDropdown

Lines 301-306 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (+16 lines)
301	</div> <!-- /.btn-group -->	301	</div> <!-- /.btn-group -->
302	[% END %]	302	[% END %]
303		303
		304	[% IF Koha.Preference('EnableSearchResultMARCExport') && Koha.Preference('SearchEngine') == 'Elasticsearch' && CAN_user_tools_export_catalog %]
		305	<div class="btn-group">
		306	<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
		307	Export results<span class="caret"></span>
		308	</button>
		309	<ul class="dropdown-menu">
		310	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=ISO2709[% PROCESS sort_search_query %]">MARC (UTF-8)</a></li>
		311	<li><a href="/cgi-bin/koha/catalogue/search.pl?count=[% results_per_page \| uri %]&export=1&export_format=MARCXML[% PROCESS sort_search_query %]">MARC XML</a></li>
		312	</ul>
		313	</div> <!-- /.btn-group -->
		314	[% END %]
		315
304	</div> <!-- /#selection_ops -->	316	</div> <!-- /#selection_ops -->
305	</div> <!-- /#searchheader -->	317	</div> <!-- /#searchheader -->
306		318
Lines 325-330 Link Here
325	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>	337	<div class="dialog alert"><p><strong>Error:</strong> [% query_error \| html %]</p></div>
326	[% END %]	338	[% END %]
327		339
		340	[% IF ( export_user_email ) %]
		341	<div class="dialog message">Export in progress, an email will results will be sent to [% export_user_email \| html %]</div>
		342	[% END %]
		343
328	<!-- Search Results Table -->	344	<!-- Search Results Table -->
329	[% IF ( total ) %]	345	[% IF ( total ) %]
330	[% IF ( scan ) %]	346	[% IF ( scan ) %]





subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 62;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my @mappings = (
        {

    ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
    is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");


    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');

    $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
    is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");

    # Search export functionality
    # Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
    my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);

    for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {

        t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);

        $docs = $see->marc_records_to_documents(\@source_docs);

        # Emulate Elasticsearch response docs structure
        my @es_response_docs = map { { _source => $_ } } @{$docs};

        my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');

        # $large_marc_record should not have been encoded as ISO2709
        # since exceeds maximum size, see above
        my @tmp = ($marc_record_1, $marc_record_2);
        is(
            $records_data->{ISO2709},
            join('', map { $_->as_usmarc() } @tmp),
            "ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
        );

        my $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            MARC::File::XML::record($large_marc_record, 'MARC21'),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
        );

        $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');

        $expected_marc_xml = join("\n",
            MARC::File::XML::header(),
            join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
            MARC::File::XML::footer()
        );

        is(
            $records_data->{MARCXML},
            $expected_marc_xml,
            "MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
        );

    }

    push @mappings, {
        name => 'title',
        type => 'string',


    is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);

    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");

    plan tests => 2;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');

    my $builder = t::lib::TestBuilder->new;
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
- 

Return to bug 27859

Lines 131-136 Searching: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+13 lines)
131	yes: use	131	yes: use
132	no: "don't use"	132	no: "don't use"
133	- 'the operator "phr" in the callnumber and standard number staff client searches'	133	- 'the operator "phr" in the callnumber and standard number staff client searches'
		134	-
		135	- pref: EnableSearchResultMARCExport
		136	type: boolean
		137	default: yes
		138	choices:
		139	yes: Enable
		140	no: Disable
		141	- "MARC export of search results. The export will be sent to to the logged in user's email address. Records exceeding the ISO2709 record size will be send at separate MARC XML attachment regardless of chosen export format (ElasticSearch only)."
		142	-
		143	- "Limit exported MARC records from search results to a maximum of"
		144	- pref: SearchResultMARCExportLimit
		145	class: integer
		146	- "records."
134	Results display:	147	Results display:
135	-	148	-
136	- pref: numSearchResultsDropdown	149	- pref: numSearchResultsDropdown

Lines 41-48 use YAML::Syck; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-33 / +181 lines)
41		41
42	use List::Util qw( sum0 reduce );	42	use List::Util qw( sum0 reduce );
43	use MARC::File::XML;	43	use MARC::File::XML;
44	use MIME::Base64;	44	use MIME::Base64 qw(encode_base64 decode_base64);
45	use Encode qw(encode);	45	use Encode qw(encode decode);
46	use Business::ISBN;	46	use Business::ISBN;
47	use Scalar::Util qw(looks_like_number);	47	use Scalar::Util qw(looks_like_number);
48		48
Lines 524-530 sub marc_records_to_documents { Link Here
524	my $control_fields_rules = $rules->{control_fields};	524	my $control_fields_rules = $rules->{control_fields};
525	my $data_fields_rules = $rules->{data_fields};	525	my $data_fields_rules = $rules->{data_fields};
526	my $marcflavour = lc C4::Context->preference('marcflavour');	526	my $marcflavour = lc C4::Context->preference('marcflavour');
527	my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
528		527
529	my @record_documents;	528	my @record_documents;
530		529
Lines 679-716 sub marc_records_to_documents { Link Here
679	}	678	}
680	}	679	}
681	}	680	}
		681	my $preferred_format = C4::Context->preference('ElasticsearchMARCFormat');
682		682
683	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	683	my ($encoded_record, $format) = $self->search_document_marc_record_encode(
684	$record->encoding('UTF-8');	684	$record,
685	if ($use_array) {	685	$preferred_format,
686	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);	686	$marcflavour
687	$record_document->{'marc_format'} = 'ARRAY';	687	);
		688
		689	if ($preferred_format eq 'ARRAY') {
		690	$record_document->{'marc_data_array'} = $encoded_record;
688	} else {	691	} else {
689	my @warnings;	692	$record_document->{'marc_data'} = $encoded_record;
690	{	693	}
691	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)	694	$record_document->{'marc_format'} = $format;
692	local $SIG{__WARN__} = sub {	695
693	push @warnings, $_[0];	696	push @record_documents, $record_document;
694	};	697	}
695	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));	698	return \@record_documents;
696	}	699	}
697	if (@warnings) {	700
698	# Suppress warnings if record length exceeded	701	=head2 search_document_marc_record_encode($record, $format, $marcflavour)
699	unless (substr($record->leader(), 0, 5) eq '99999') {	702	my ($encoded_record, $format) = search_document_marc_record_encode($record, $format, $marcflavour)
700	foreach my $warning (@warnings) {	703
701	carp $warning;	704	Encode a MARC::Record to the prefered marc document record format. If record exceeds ISO2709 maximum
702	}	705	size record size and C<$format> is set to 'base64ISO2709' format will fallback to 'MARCXML' instead.
		706
		707	=over 4
		708
		709	=item C<$record>
		710
		711	A MARC::Record object
		712
		713	=item C<$marcflavour>
		714
		715	The marcflavour to use
		716
		717	=back
		718
		719	=cut
		720
		721	sub search_document_marc_record_encode {
		722	my ($self, $record, $format, $marcflavour) = @_;
		723
		724	$record->encoding('UTF-8');
		725
		726	if ($format eq 'ARRAY') {
		727	return ($self->_marc_to_array($record), $format);
		728	}
		729	elsif ($format eq 'base64ISO2709' \|\| $format eq 'ISO2709') {
		730	my @warnings;
		731	my $marc_data;
		732	{
		733	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
		734	local $SIG{__WARN__} = sub {
		735	push @warnings, $_[0];
		736	};
		737	$marc_data = $record->as_usmarc();
		738	}
		739	if (@warnings) {
		740	# Suppress warnings if record length exceeded
		741	unless (substr($record->leader(), 0, 5) eq '99999') {
		742	foreach my $warning (@warnings) {
		743	carp $warning;
703	}	744	}
704	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
705	$record_document->{'marc_format'} = 'MARCXML';
706	}	745	}
707	else {	746	return (MARC::File::XML::record($record, $marcflavour), 'MARCXML');
708	$record_document->{'marc_format'} = 'base64ISO2709';	747	}
		748	else {
		749	if ($format eq 'base64ISO2709') {
		750	$marc_data = encode_base64(encode('UTF-8', $marc_data));
709	}	751	}
		752	return ($marc_data, $format);
710	}	753	}
711	push @record_documents, $record_document;
712	}	754	}
713	return \@record_documents;	755	elsif ($format eq 'MARCXML') {
		756	return (MARC::File::XML::record($record, $marcflavour), $format);
		757	}
		758	else {
		759	# This should be unlikely to happen
		760	croak "Invalid marc record serialization format: $format";
		761	}
		762	}
		763
		764	=head2 search_document_marc_record_decode
		765	my $marc_record = $self->search_document_marc_record_decode(@result);
		766
		767	Extract marc data from Elasticsearch result and decode to MARC::Record object
		768
		769	=cut
		770
		771	sub search_document_marc_record_decode {
		772	# Result is passed in as array, will get flattened
		773	# and first element will be $result
		774	my ($self, $result) = @_;
		775	if ($result->{marc_format} eq 'base64ISO2709') {
		776	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
		777	}
		778	elsif ($result->{marc_format} eq 'MARCXML') {
		779	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
		780	}
		781	elsif ($result->{marc_format} eq 'ARRAY') {
		782	return $self->_array_to_marc($result->{marc_data_array});
		783	}
		784	else {
		785	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
		786	}
		787	}
		788
		789	=head2 search_document_marc_records_encode_from_docs($docs, $preferred_format)
		790
		791	$records_data = $self->search_document_marc_records_encode_from_docs($docs, $preferred_format)
		792
		793	Return marc encoded records from ElasticSearch search result documents. The return value
		794	C<$marc_records> is a hashref with encoded records keyed by MARC format.
		795
		796	=over 4
		797
		798	=item C<$docs>
		799
		800	An arrayref of Elasticsearch search documents
		801
		802	=item C<$preferred_format>
		803
		804	The preferred marc format: 'MARCXML' or 'ISO2709'. Records exceeding maximum
		805	length supported by ISO2709 will be exported as 'MARCXML' even if C<$preferred_format>
		806	is set to 'ISO2709'.
		807
		808	=back
		809
		810	=cut
		811
		812	sub search_document_marc_records_encode_from_docs {
		813
		814	my ($self, $docs, $preferred_format) = @_;
		815
		816	my %encoded_records = (
		817	'ISO2709' => [],
		818	'MARCXML' => []
		819	);
820
821	unless (exists $encoded_records{$preferred_format}) {
822	croak "Invalid preferred format: $preferred_format";
823	}
824
825	for my $es_record (@{$docs}) {
826	# Special optimized cases
827	my $marc_data;
828	my $resulting_format = $preferred_format;
829	if ($preferred_format eq 'MARCXML' && $es_record->{_source}{marc_format} eq 'MARCXML') {
830	$marc_data = $es_record->{_source}{marc_data};
831	}
832	elsif ($preferred_format eq 'ISO2709' && $es_record->{_source}->{marc_format} eq 'base64ISO2709') {
833	# Stored as UTF-8 encoded binary in index, so needs to be decoded
834	$marc_data = decode('UTF-8', decode_base64($es_record->{_source}->{marc_data}));
835	}
836	else {
837	my $record = $self->search_document_marc_record_decode($es_record->{'_source'});
838	my $marcflavour = lc C4::Context->preference('marcflavour');
839	($marc_data, $resulting_format) = $self->search_document_marc_record_encode($record, $preferred_format, $marcflavour);
840	}
841	push @{$encoded_records{$resulting_format}}, $marc_data;
842	}
843	if (@{$encoded_records{'ISO2709'}}) {
844	$encoded_records{'ISO2709'} = join("", @{$encoded_records{'ISO2709'}});
845	}
846	else {
847	delete $encoded_records{'ISO2709'};
848	}
849
850	if (@{$encoded_records{'MARCXML'}}) {
851	$encoded_records{'MARCXML'} = join("\n",
852	MARC::File::XML::header(),
853	join("\n", @{$encoded_records{'MARCXML'}}),
854	MARC::File::XML::footer()
855	);
856	}
857	else {
858	delete $encoded_records{'MARCXML'};
859	}
860
861	return \%encoded_records;
714	}	862	}
715		863
716	=head2 _marc_to_array($record)	864	=head2 _marc_to_array($record)
Lines 784-801 sub _array_to_marc { Link Here
784	$record->leader($data->{leader});	932	$record->leader($data->{leader});
785	for my $field (@{$data->{fields}}) {	933	for my $field (@{$data->{fields}}) {
786	my $tag = (keys %{$field})[0];	934	my $tag = (keys %{$field})[0];
787	$field = $field->{$tag};	935	my $field_data = $field->{$tag};
788	my $marc_field;	936	my $marc_field;
789	if (ref($field) eq 'HASH') {	937	if (ref($field_data) eq 'HASH') {
790	my @subfields;	938	my @subfields;
791	foreach my $subfield (@{$field->{subfields}}) {	939	foreach my $subfield (@{$field_data->{subfields}}) {
792	my $code = (keys %{$subfield})[0];	940	my $code = (keys %{$subfield})[0];
793	push @subfields, $code;	941	push @subfields, $code;
794	push @subfields, $subfield->{$code};	942	push @subfields, $subfield->{$code};
795	}	943	}
796	$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);	944	$marc_field = MARC::Field->new($tag, $field_data->{ind1}, $field_data->{ind2}, @subfields);
797	} else {	945	} else {
798	$marc_field = MARC::Field->new($tag, $field)	946	$marc_field = MARC::Field->new($tag, $field_data)
799	}	947	}
800	$record->append_fields($marc_field);	948	$record->append_fields($marc_field);
801	}	949	}

Lines 50-58 use Koha::SearchEngine::Search; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-31 / +3 lines)
50	use Koha::Exceptions::Elasticsearch;	50	use Koha::Exceptions::Elasticsearch;
51	use MARC::Record;	51	use MARC::Record;
52	use MARC::File::XML;	52	use MARC::File::XML;
53	use Data::Dumper; #TODO remove
54	use Carp qw(cluck);
55	use MIME::Base64;
56		53
57	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));	54	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
58		55
Lines 166-172 sub search_compat { Link Here
166	my $index = $offset;	163	my $index = $offset;
167	my $hits = $results->{'hits'};	164	my $hits = $results->{'hits'};
168	foreach my $es_record (@{$hits->{'hits'}}) {	165	foreach my $es_record (@{$hits->{'hits'}}) {
169	$records[$index++] = $self->decode_record_from_result($es_record->{'_source'});	166	$records[$index++] = $self->search_document_marc_record_decode($es_record->{'_source'});
170	}	167	}
171		168
172	# consumers of this expect a name-spaced result, we provide the default	169	# consumers of this expect a name-spaced result, we provide the default
Lines 227-233 sub search_auth_compat { Link Here
227	# it's not reproduced here yet.	224	# it's not reproduced here yet.
228	my $authtype = $rs->single;	225	my $authtype = $rs->single;
229	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";	226	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
230	my $marc = $self->decode_record_from_result($record);	227	my $marc = $self->search_document_marc_record_decode($record);
231	my $mainentry = $marc->field($auth_tag_to_report);	228	my $mainentry = $marc->field($auth_tag_to_report);
232	my $reported_tag;	229	my $reported_tag;
233	if ($mainentry) {	230	if ($mainentry) {
Lines 347-353 sub simple_search_compat { Link Here
347	my @records;	344	my @records;
348	my $hits = $results->{'hits'};	345	my $hits = $results->{'hits'};
349	foreach my $es_record (@{$hits->{'hits'}}) {	346	foreach my $es_record (@{$hits->{'hits'}}) {
350	push @records, $self->decode_record_from_result($es_record->{'_source'});	347	push @records, $self->search_document_marc_record_decode($es_record->{'_source'});
351	}	348	}
352	return (undef, \@records, $hits->{'total'});	349	return (undef, \@records, $hits->{'total'});
353	}	350	}
Lines 367-397 sub extract_biblionumber { Link Here
367	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );	364	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
368	}	365	}
369		366
370	=head2 decode_record_from_result
371	my $marc_record = $self->decode_record_from_result(@result);
372
373	Extracts marc data from Elasticsearch result and decodes to MARC::Record object
374
375	=cut
376
377	sub decode_record_from_result {
378	# Result is passed in as array, will get flattened
379	# and first element will be $result
380	my ( $self, $result ) = @_;
381	if ($result->{marc_format} eq 'base64ISO2709') {
382	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
383	}
384	elsif ($result->{marc_format} eq 'MARCXML') {
385	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
386	}
387	elsif ($result->{marc_format} eq 'ARRAY') {
388	return $self->_array_to_marc($result->{marc_data_array});
389	}
390	else {
391	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
392	}
393	}
394
395	=head2 max_result_window	367	=head2 max_result_window
396		368
397	Returns the maximum number of results that can be fetched	369	Returns the maximum number of results that can be fetched

Lines 175-181 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/sysprefs.sql (-1 / +3 lines)
175	('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),	175	('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
176	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),	176	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
177	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),	177	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
178	('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709\|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),	178	('ElasticsearchMARCFormat', 'base64ISO2709', 'base64ISO2709\|ARRAY', 'Elasticsearch MARC format. base64ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
179	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),	179	('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
180	('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),	180	('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
181	('EmailPurchaseSuggestions','0','0\|EmailAddressForSuggestions\|BranchEmailAddress\|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'),	181	('EmailPurchaseSuggestions','0','0\|EmailAddressForSuggestions\|BranchEmailAddress\|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'),
Lines 184-189 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
184	('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),	184	('EnableOpacSearchHistory','1','YesNo','Enable or disable opac search history',''),
185	('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),	185	('EnablePointOfSale','0',NULL,'Enable the point of sale feature to allow anonymous transactions with the accounting system. (Requires UseCashRegisters)','YesNo'),
186	('EnableSearchHistory','0','','Enable or disable search history','YesNo'),	186	('EnableSearchHistory','0','','Enable or disable search history','YesNo'),
		187	('EnableSearchResultMARCExport', '1', '', 'Enable search result MARC export', 'YesNo'),
187	('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),	188	('EnhancedMessagingPreferences','1','','If ON, allows patrons to select to receive additional messages about items due or nearly due.','YesNo'),
188	('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),	189	('EnhancedMessagingPreferencesOPAC', '1', NULL, 'If ON, show patrons messaging setting on the OPAC.', 'YesNo'),
189	('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),	190	('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'),
Lines 569-574 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
569	('SCOUserJS','',NULL,'Define custom javascript for inclusion in the SCO module','free'),	570	('SCOUserJS','',NULL,'Define custom javascript for inclusion in the SCO module','free'),
570	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),	571	('SearchEngine','Zebra','Elasticsearch\|Zebra','Search Engine','Choice'),
571	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),	572	('SearchMyLibraryFirst','0',NULL,'If ON, OPAC searches return results limited by the user\'s library by default if they are logged in','YesNo'),
		573	('SearchResultMARCExportLimit', NULL, NULL, 'Search result MARC export limit', 'integer'),
572	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),	574	('SearchWithISBNVariations','0',NULL,'If enabled, search on all variations of the ISBN','YesNo'),
573	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),	575	('SelfCheckAllowByIPRanges','',NULL,'(Leave blank if not used. Use ranges or simple ip addresses separated by spaces, like <code>192.168.1.1 192.168.0.0/24</code>.)','Short'),
574	('SelfCheckHelpMessage','','70\|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),	576	('SelfCheckHelpMessage','','70\|10','Enter HTML to include under the basic Web-based Self Checkout instructions on the Help page','Textarea'),

Lines 122-131 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-7 / +58 lines)
122		122
123	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	123	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
124		124
125	plan tests => 53;	125	plan tests => 62;
126		126
127	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	127	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
128	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	128	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
129		129
130	my @mappings = (	130	my @mappings = (
131	{	131	{
Lines 420-426 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
420	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');	420	ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
421	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');	421	is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
422		422
423	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	423	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
424		424
425	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");	425	ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
426	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");	426	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
Lines 511-521 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
511		511
512	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');	512	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
513		513
514	$decoded_marc_record = $see->decode_record_from_result($docs->[0]);	514	$decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
515		515
516	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");	516	ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
517	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");	517	is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
518		518
		519	# Search export functionality
		520	# Koha::SearchEngine::Elasticsearch::search_document_marc_records_encode_from_docs()
		521	my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record);
		522
		523	for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') {
		524
		525	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format);
		526
		527	$docs = $see->marc_records_to_documents(\@source_docs);
		528
		529	# Emulate Elasticsearch response docs structure
		530	my @es_response_docs = map { { _source => $_ } } @{$docs};
		531
		532	my $records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'ISO2709');
		533
		534	# $large_marc_record should not have been encoded as ISO2709
		535	# since exceeds maximum size, see above
		536	my @tmp = ($marc_record_1, $marc_record_2);
		537	is(
		538	$records_data->{ISO2709},
		539	join('', map { $_->as_usmarc() } @tmp),
		540	"ISO2709 encoded records from ElasticSearch result are identical with source records using index format \"$es_marc_format\""
		541	);
		542
		543	my $expected_marc_xml = join("\n",
		544	MARC::File::XML::header(),
		545	MARC::File::XML::record($large_marc_record, 'MARC21'),
		546	MARC::File::XML::footer()
		547	);
		548
		549	is(
		550	$records_data->{MARCXML},
		551	$expected_marc_xml,
		552	"Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is indentical with source record using index format \"$es_marc_format\""
		553	);
		554
		555	$records_data = $see->search_document_marc_records_encode_from_docs(\@es_response_docs, 'MARCXML');
		556
		557	$expected_marc_xml = join("\n",
		558	MARC::File::XML::header(),
		559	join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs),
		560	MARC::File::XML::footer()
		561	);
		562
		563	is(
		564	$records_data->{MARCXML},
		565	$expected_marc_xml,
		566	"MARCXML encoded records from ElasticSearch result are indentical with source records using index format \"$es_marc_format\""
		567	);
		568
		569	}
		570
519	push @mappings, {	571	push @mappings, {
520	name => 'title',	572	name => 'title',
521	type => 'string',	573	type => 'string',
Lines 633-639 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
633		685
634	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');	686	is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
635		687
636	my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);	688	my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]);
637		689
638	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");	690	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
639	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");	691	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
Lines 644-650 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
644	plan tests => 2;	696	plan tests => 2;
645		697
646	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	698	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
647	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	699	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'base64ISO2709');
648		700
649	my $builder = t::lib::TestBuilder->new;	701	my $builder = t::lib::TestBuilder->new;
650	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{	702	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
651	-