Attachment #174320 for bug #38270

View | Details | Raw Unified | Return to bug 38270
Collapse All | Expand All




use Encode qw( encode );
use Business::ISBN;
use Scalar::Util qw( looks_like_number );
use Compress::Zlib qw/deflateInit Z_BEST_COMPRESSION/;

__PACKAGE__->mk_ro_accessors(qw( index index_name ));
__PACKAGE__->mk_accessors(qw( sort_fields ));

    my $control_fields_rules = $rules->{control_fields};
    my $data_fields_rules = $rules->{data_fields};
    my $marcflavour = lc C4::Context->preference('marcflavour');
    my $marcformat_syspref = C4::Context->preference('ElasticsearchMARCFormat');

    my @record_documents;



        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
        $record->encoding('UTF-8');
        if ($marcformat_syspref eq 'ARRAY') {
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
            $record_document->{'marc_format'}     = 'ARRAY';
        } elsif ($marcformat_syspref eq 'MARCXML'){
            $record_document->{'marc_data'}   = $record->as_xml_record($marcflavour);
            $record_document->{'marc_format'} = 'MARCXML';
        } elsif ($marcformat_syspref eq 'MARCXML_COMPRESSED'){
            my $deflate = deflateInit( -Level => Z_BEST_COMPRESSION );
            my $output  = "";
            if ($deflate) {
                my $input_xml = $record->as_xml_record($marcflavour);
                if ($input_xml) {
                    my $compressed = $deflate->deflate($input_xml);
                    $output .= $compressed;
                    my $final = $deflate->flush();
                    $output .= $final;
                }
            }
            $record_document->{'marc_data'}   = encode_base64($output);
            $record_document->{'marc_format'} = 'MARCXML_COMPRESSED';
        } else {
            my @warnings;
            {

Lines 53-58 use MARC::Record; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+6 lines)
53	use MARC::File::XML;	53	use MARC::File::XML;
54	use MIME::Base64 qw( decode_base64 );	54	use MIME::Base64 qw( decode_base64 );
55	use JSON;	55	use JSON;
		56	use Compress::Zlib qw/inflateInit/;
56		57
57	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));	58	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
58		59
Lines 399-404 sub decode_record_from_result { Link Here
399	elsif ($result->{marc_format} eq 'MARCXML') {	400	elsif ($result->{marc_format} eq 'MARCXML') {
400	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));	401	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
401	}	402	}
		403	elsif ($result->{marc_format} eq 'MARCXML_COMPRESSED') {
		404	my $inflate = inflateInit();
		405	my $marcxml = $inflate->inflate( decode_base64( $result->{marc_data} ) );
		406	return MARC::Record->new_from_xml( $marcxml, 'UTF-8', uc C4::Context->preference('marcflavour') );
		407	}
402	elsif ($result->{marc_format} eq 'ARRAY') {	408	elsif ($result->{marc_format} eq 'ARRAY') {
403	return $self->_array_to_marc($result->{marc_data_array});	409	return $self->_array_to_marc($result->{marc_data_array});
404	}	410	}

Lines 308-312 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (-1 / +2 lines)
308	choices:	308	choices:
309	"ISO2709": "ISO2709 (exchange format)"	309	"ISO2709": "ISO2709 (exchange format)"
310	"ARRAY": "Searchable array"	310	"ARRAY": "Searchable array"
		311	"MARCXML": "MARCXML"
		312	"MARCXML_COMPRESSED": "MARCXML compressed using DEFLATE algorithm"
311	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.	313	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
312	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.	314	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
313	-

Return to bug 38270

Lines 47-52 use MIME::Base64 qw( encode_base64 ); Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-3 / +21 lines)
47	use Encode qw( encode );	47	use Encode qw( encode );
48	use Business::ISBN;	48	use Business::ISBN;
49	use Scalar::Util qw( looks_like_number );	49	use Scalar::Util qw( looks_like_number );
		50	use Compress::Zlib qw/deflateInit Z_BEST_COMPRESSION/;
50		51
51	__PACKAGE__->mk_ro_accessors(qw( index index_name ));	52	__PACKAGE__->mk_ro_accessors(qw( index index_name ));
52	__PACKAGE__->mk_accessors(qw( sort_fields ));	53	__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 564-570 sub marc_records_to_documents { Link Here
564	my $control_fields_rules = $rules->{control_fields};	565	my $control_fields_rules = $rules->{control_fields};
565	my $data_fields_rules = $rules->{data_fields};	566	my $data_fields_rules = $rules->{data_fields};
566	my $marcflavour = lc C4::Context->preference('marcflavour');	567	my $marcflavour = lc C4::Context->preference('marcflavour');
567	my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';	568	my $marcformat_syspref = C4::Context->preference('ElasticsearchMARCFormat');
568		569
569	my @record_documents;	570	my @record_documents;
570		571
Lines 805-813 sub marc_records_to_documents { Link Here
805		806
806	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	807	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
807	$record->encoding('UTF-8');	808	$record->encoding('UTF-8');
808	if ($use_array) {	809	if ($marcformat_syspref eq 'ARRAY') {
809	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);	810	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);
810	$record_document->{'marc_format'} = 'ARRAY';	811	$record_document->{'marc_format'} = 'ARRAY';
		812	} elsif ($marcformat_syspref eq 'MARCXML'){
		813	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
		814	$record_document->{'marc_format'} = 'MARCXML';
		815	} elsif ($marcformat_syspref eq 'MARCXML_COMPRESSED'){
		816	my $deflate = deflateInit( -Level => Z_BEST_COMPRESSION );
		817	my $output = "";
		818	if ($deflate) {
		819	my $input_xml = $record->as_xml_record($marcflavour);
		820	if ($input_xml) {
		821	my $compressed = $deflate->deflate($input_xml);
		822	$output .= $compressed;
		823	my $final = $deflate->flush();
		824	$output .= $final;
		825	}
		826	}
		827	$record_document->{'marc_data'} = encode_base64($output);
		828	$record_document->{'marc_format'} = 'MARCXML_COMPRESSED';
811	} else {	829	} else {
812	my @warnings;	830	my @warnings;
813	{	831	{