From f0467da5d4e2de8bd4216e4b21fab30b2be6ad38 Mon Sep 17 00:00:00 2001 From: David Cook Date: Mon, 11 Nov 2024 05:35:28 +0000 Subject: [PATCH] Bug 38270: Add MARCXML and MARCXML_COMPRESSED to ElasticsearchMARCFormat syspref This change adds MARCXML and MARCXML_COMPRESSED to the ElasticsearchMARCFormat system preference. Previously, MARCXML was a fallback format for base64ISO2709, but this provides it as a valid choice in itself. For storage and performance reasons, MARCXML_COMPRESSED is another alternative. It uses the DEFLATE algorithm using the zlib library via C bindings. It is just as fast as base64ISO2709 for indexing, and it's actually superior in terms of storage, especially for large records which fallback to MARCXML. Test plan: 0. Apply the patch 0b. Set up koha-testing-docker to use Elasticsearch 0c. koha-plack --restart kohadev 1. Go to http://localhost:8081/cgi-bin/koha/admin/preferences.pl? op=search&searchfield=ElasticsearchMARCFormat 2. Switch the setting to "ISO2709" 3. Create a MARC record with a field over 9999 characters 4. Create a MARC record with a record length over 99999 (but individual field length of under 32000 characters) 5. Reindex the database 6. Search for the records (using a keyword that matches these and other records) 7. Note that the record with a field over 9999 characters is not properly retrieved 8. Look on the detail page at "Elastic record" and note that the records use a "marc_format" of "baseISO2709" and "MARCXML" respectively 9. Change "ElasticsearchMARCFormat" to "MARCXML" 10. Reindex the database 11. Note that the records now use a marc_format of "MARCXML" and both records are now displaying properly in search 12. Change "ElasticsearchMARCFormat" to "MARCXML compressed using DEFLATE algorithm" 13. Reindex the database 14. Note that the records now use a marc_format of "MARCXML_COMPRESSED" and both records are now displaying properly in search 15. prove -v t/db_dependent/Koha/SearchEngine/Elasticsearch.t *Bonus points* - Create a record with 100 items with the same large note fields - Using "ISO2709" note that the fallback is to "MARCXML" and the marc_data field is very large - Change to "MARCXML compressed using DEFLATE algorithm" and note that the marc_data field is very small. Signed-off-by: Martin Renvoize --- Koha/SearchEngine/Elasticsearch.pm | 24 ++++++++++++++++--- Koha/SearchEngine/Elasticsearch/Search.pm | 6 +++++ .../en/modules/admin/preferences/admin.pref | 2 ++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 5ef52357202..ee5d9b66143 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -47,6 +47,7 @@ use MIME::Base64 qw( encode_base64 ); use Encode qw( encode ); use Business::ISBN; use Scalar::Util qw( looks_like_number ); +use Compress::Zlib qw/deflateInit Z_BEST_COMPRESSION/; __PACKAGE__->mk_ro_accessors(qw( index index_name )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -564,7 +565,7 @@ sub marc_records_to_documents { my $control_fields_rules = $rules->{control_fields}; my $data_fields_rules = $rules->{data_fields}; my $marcflavour = lc C4::Context->preference('marcflavour'); - my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY'; + my $marcformat_syspref = C4::Context->preference('ElasticsearchMARCFormat'); my @record_documents; @@ -805,9 +806,26 @@ sub marc_records_to_documents { # TODO: Perhaps should check if $records_document non empty, but really should never be the case $record->encoding('UTF-8'); - if ($use_array) { + if ($marcformat_syspref eq 'ARRAY') { $record_document->{'marc_data_array'} = $self->_marc_to_array($record); - $record_document->{'marc_format'} = 'ARRAY'; + $record_document->{'marc_format'} = 'ARRAY'; + } elsif ($marcformat_syspref eq 'MARCXML'){ + $record_document->{'marc_data'} = $record->as_xml_record($marcflavour); + $record_document->{'marc_format'} = 'MARCXML'; + } elsif ($marcformat_syspref eq 'MARCXML_COMPRESSED'){ + my $deflate = deflateInit( -Level => Z_BEST_COMPRESSION ); + my $output = ""; + if ($deflate) { + my $input_xml = $record->as_xml_record($marcflavour); + if ($input_xml) { + my $compressed = $deflate->deflate($input_xml); + $output .= $compressed; + my $final = $deflate->flush(); + $output .= $final; + } + } + $record_document->{'marc_data'} = encode_base64($output); + $record_document->{'marc_format'} = 'MARCXML_COMPRESSED'; } else { my @warnings; { diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index 171c00c729d..fbaefd1b662 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -53,6 +53,7 @@ use MARC::Record; use MARC::File::XML; use MIME::Base64 qw( decode_base64 ); use JSON; +use Compress::Zlib qw/inflateInit/; Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store )); @@ -413,6 +414,11 @@ sub decode_record_from_result { elsif ($result->{marc_format} eq 'MARCXML') { return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour')); } + elsif ($result->{marc_format} eq 'MARCXML_COMPRESSED') { + my $inflate = inflateInit(); + my $marcxml = $inflate->inflate( decode_base64( $result->{marc_data} ) ); + return MARC::Record->new_from_xml( $marcxml, 'UTF-8', uc C4::Context->preference('marcflavour') ); + } elsif ($result->{marc_format} eq 'ARRAY') { return $self->_array_to_marc($result->{marc_data_array}); } diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref index b44075a9b01..d6f080a1279 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref @@ -308,5 +308,7 @@ Administration: choices: "ISO2709": "ISO2709 (exchange format)" "ARRAY": "Searchable array" + "MARCXML": "MARCXML" + "MARCXML_COMPRESSED": "MARCXML compressed using DEFLATE algorithm" -
ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable. -
NOTE: Making the full record searchable may have a negative effect on relevance ranking of search results. -- 2.47.0