Attachment #87436 for bug #22258

View | Details | Raw Unified | Return to bug 22258
Collapse All | Expand All




    my $control_fields_rules = $rules->{control_fields};
    my $data_fields_rules = $rules->{data_fields};
    my $marcflavour = lc C4::Context->preference('marcflavour');
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';

    my @record_documents;



        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
        $record->encoding('UTF-8');
        if ($use_array) {
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
            $record_document->{'marc_format'} = 'ARRAY';
        } else {
            my @warnings;
            {
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
                local $SIG{__WARN__} = sub {
                    push @warnings, $_[0];
                };
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
            }
            if (@warnings) {
                # Suppress warnings if record length exceeded
                unless (substr($record->leader(), 0, 5) eq '99999') {
                    foreach my $warning (@warnings) {
                        carp $warning;
                    }
                }
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
                $record_document->{'marc_format'} = 'MARCXML';
            }
            else {
                $record_document->{'marc_format'} = 'base64ISO2709';
            }
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
            $record_document->{'marc_format'} = 'MARCXML';
        }
        else {
            $record_document->{'marc_format'} = 'base64ISO2709';
        }
        my $id = $record->subfield('999', 'c');
        push @record_documents, [$id, $record_document];

    return \@record_documents;
}

=head2 _marc_to_array($record)

    my @fields = _marc_to_array($record)

Convert a MARC::Record to an array modeled after MARC-in-JSON
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)

=over 4

=item C<$record>

A MARC::Record object

=back

=cut

sub _marc_to_array {
    my ($self, $record) = @_;

    my $data = {
        leader => $record->leader(),
        fields => []
    };
    for my $field ($record->fields()) {
        my $tag = $field->tag();
        if ($field->is_control_field()) {
            push @{$data->{fields}}, {$tag => $field->data()};
        } else {
            my $subfields = ();
            foreach my $subfield ($field->subfields()) {
                my ($code, $contents) = @{$subfield};
                push @{$subfields}, {$code => $contents};
            }
            push @{$data->{fields}}, {
                $tag => {
                    ind1 => $field->indicator(1),
                    ind2 => $field->indicator(2),
                    subfields => $subfields
                }
            };
        }
    }
    return $data;
}

=head2 _array_to_marc($data)

    my $record = _array_to_marc($data)

Convert an array modeled after MARC-in-JSON to a MARC::Record

=over 4

=item C<$data>

An array modeled after MARC-in-JSON
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)

=back

=cut

sub _array_to_marc {
    my ($self, $data) = @_;

    my $record = MARC::Record->new();

    $record->leader($data->{leader});
    for my $field (@{$data->{fields}}) {
        my $tag = (keys %{$field})[0];
        $field = $field->{$tag};
        my $marc_field;
        if (ref($field) eq 'HASH') {
            my @subfields;
            foreach my $subfield (@{$field->{subfields}}) {
                my $code = (keys %{$subfield})[0];
                push @subfields, $code;
                push @subfields, $subfield->{$code};
            }
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
        } else {
            $marc_field = MARC::Field->new($tag, $field)
        }
        $record->append_fields($marc_field);
    }
;
    return $record;
}

=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)

    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)

Lines 383-388 sub decode_record_from_result { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+3 lines)
383	elsif ($result->{marc_format} eq 'MARCXML') {	383	elsif ($result->{marc_format} eq 'MARCXML') {
384	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));	384	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
385	}	385	}
		386	elsif ($result->{marc_format} eq 'ARRAY') {
		387	return $self->_array_to_marc($result->{marc_data_array});
		388	}
386	else {	389	else {
387	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");	390	Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
388	}	391	}

Lines 10-15 general: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+3 lines)
10	type: text	10	type: text
11	analyzer: keyword	11	analyzer: keyword
12	index: false	12	index: false
		13	marc_data_array:
		14	type: object
		15	dynamic: true
13	marc_format:	16	marc_format:
14	store: true	17	store: true
15	type: text	18	type: text

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_22258.perl (+11 lines)
	1	$DBversion = 'XXX'; # will be replaced by the RM
	2	if( CheckVersion( $DBversion ) ) {
	3	$dbh->do(q{
	4	INSERT IGNORE INTO `systempreferences` (`variable`,`value`,`explanation`,`options`,`type`) VALUES
	5	('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709\|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice')
	6	});
	7
	8	# Always end with this (adjust the bug info)
	9	SetVersion( $DBversion );
	10	print "Upgrade to $DBversion done (Bug 22258 - Add ElasticsearchMARCFormat preference)\n";
	11	}

Lines 157-162 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/sysprefs.sql (+1 lines)
157	('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),	157	('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
158	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),	158	('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
159	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),	159	('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
		160	('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709\|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
160	('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),	161	('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
161	('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),	162	('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
162	('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),	163	('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),




              choices:
                Zebra: Zebra
                Elasticsearch: Elasticsearch
        -
            - "Elasticsearch MARC format: "
            - pref: ElasticsearchMARCFormat
              default: "ISO2709"
              choices:
                "ISO2709": "ISO2709 (exchange format)"
                "ARRAY": "Searchable array"
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.





use Modern::Perl;

use Test::More tests => 5;
use Test::Exception;

use t::lib::Mocks;

    plan tests => 49;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');

    my @mappings = (
        {

    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
};

subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {

    plan tests => 6;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');

    my @mappings = (
        {
            name => 'control_number',
            type => 'string',
            facet => 0,
            suggestible => 0,
            sort => undef,
            marc_type => 'marc21',
            marc_field => '001',
        }
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->mock('_foreach_mapping', sub {
        my ($self, $sub) = @_;

        foreach my $map (@mappings) {
            $sub->(
                $map->{name},
                $map->{type},
                $map->{facet},
                $map->{suggestible},
                $map->{sort},
                $map->{marc_type},
                $map->{marc_field}
            );
        }
    });

    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new('001', '123'),
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
        MARC::Field->new('100', '', '', a => 'Author 1'),
        MARC::Field->new('110', '', '', a => 'Corp Author'),
        MARC::Field->new('210', '', '', a => 'Title 1'),
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
        MARC::Field->new('999', '', '', c => '1234567'),
    );
    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->leader('     cam  22      a 4500');
    $marc_record_2->append_fields(
        MARC::Field->new('100', '', '', a => 'Author 2'),
        # MARC::Field->new('210', '', '', a => 'Title 2'),
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
        MARC::Field->new('999', '', '', c => '1234568'),
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
    );
    my $records = [$marc_record_1, $marc_record_2];

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first

    my $docs = $see->marc_records_to_documents($records);

    # First record:
    is(scalar @{$docs}, 2, 'Two records converted to documents');

    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');

    is_deeply($docs->[0][1]->{control_number}, ['123'], 'First record control number should be set correctly');

    is($docs->[0][1]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');

    my $decoded_marc_record = $see->decode_record_from_result($docs->[0][1]);

    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
};

Return to bug 22258

Lines 416-421 sub marc_records_to_documents { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-18 / +114 lines)
416	my $control_fields_rules = $rules->{control_fields};	416	my $control_fields_rules = $rules->{control_fields};
417	my $data_fields_rules = $rules->{data_fields};	417	my $data_fields_rules = $rules->{data_fields};
418	my $marcflavour = lc C4::Context->preference('marcflavour');	418	my $marcflavour = lc C4::Context->preference('marcflavour');
		419	my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
419		420
420	my @record_documents;	421	my @record_documents;
421		422
Lines 535-560 sub marc_records_to_documents { Link Here
535		536
536	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	537	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
537	$record->encoding('UTF-8');	538	$record->encoding('UTF-8');
538	my @warnings;	539	if ($use_array) {
539	{	540	$record_document->{'marc_data_array'} = $self->_marc_to_array($record);
540	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)	541	$record_document->{'marc_format'} = 'ARRAY';
541	local $SIG{__WARN__} = sub {	542	} else {
542	push @warnings, $_[0];	543	my @warnings;
543	};	544	{
544	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));	545	# Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
545	}	546	local $SIG{__WARN__} = sub {
546	if (@warnings) {	547	push @warnings, $_[0];
547	# Suppress warnings if record length exceeded	548	};
548	unless (substr($record->leader(), 0, 5) eq '99999') {	549	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
549	foreach my $warning (@warnings) {	550	}
550	carp $warning;	551	if (@warnings) {
		552	# Suppress warnings if record length exceeded
		553	unless (substr($record->leader(), 0, 5) eq '99999') {
		554	foreach my $warning (@warnings) {
		555	carp $warning;
		556	}
551	}	557	}
		558	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
		559	$record_document->{'marc_format'} = 'MARCXML';
		560	}
		561	else {
		562	$record_document->{'marc_format'} = 'base64ISO2709';
552	}	563	}
553	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
554	$record_document->{'marc_format'} = 'MARCXML';
555	}
556	else {
557	$record_document->{'marc_format'} = 'base64ISO2709';
558	}	564	}
559	my $id = $record->subfield('999', 'c');	565	my $id = $record->subfield('999', 'c');
560	push @record_documents, [$id, $record_document];	566	push @record_documents, [$id, $record_document];
Lines 562-567 sub marc_records_to_documents { Link Here
562	return \@record_documents;	568	return \@record_documents;
563	}	569	}
564		570
		571	=head2 _marc_to_array($record)
		572
		573	my @fields = _marc_to_array($record)
		574
		575	Convert a MARC::Record to an array modeled after MARC-in-JSON
		576	(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
		577
		578	=over 4
		579
		580	=item C<$record>
		581
		582	A MARC::Record object
		583
		584	=back
		585
		586	=cut
		587
		588	sub _marc_to_array {
		589	my ($self, $record) = @_;
		590
		591	my $data = {
		592	leader => $record->leader(),
		593	fields => []
		594	};
		595	for my $field ($record->fields()) {
		596	my $tag = $field->tag();
		597	if ($field->is_control_field()) {
		598	push @{$data->{fields}}, {$tag => $field->data()};
		599	} else {
		600	my $subfields = ();
		601	foreach my $subfield ($field->subfields()) {
		602	my ($code, $contents) = @{$subfield};
		603	push @{$subfields}, {$code => $contents};
		604	}
		605	push @{$data->{fields}}, {
		606	$tag => {
		607	ind1 => $field->indicator(1),
		608	ind2 => $field->indicator(2),
		609	subfields => $subfields
		610	}
		611	};
		612	}
		613	}
		614	return $data;
		615	}
		616
		617	=head2 _array_to_marc($data)
		618
		619	my $record = _array_to_marc($data)
		620
		621	Convert an array modeled after MARC-in-JSON to a MARC::Record
		622
		623	=over 4
		624
		625	=item C<$data>
		626
		627	An array modeled after MARC-in-JSON
		628	(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
		629
		630	=back
		631
		632	=cut
		633
		634	sub _array_to_marc {
635	my ($self, $data) = @_;
636
637	my $record = MARC::Record->new();
638
639	$record->leader($data->{leader});
640	for my $field (@{$data->{fields}}) {
641	my $tag = (keys %{$field})[0];
642	$field = $field->{$tag};
643	my $marc_field;
644	if (ref($field) eq 'HASH') {
645	my @subfields;
646	foreach my $subfield (@{$field->{subfields}}) {
647	my $code = (keys %{$subfield})[0];
648	push @subfields, $code;
649	push @subfields, $subfield->{$code};
650	}
651	$marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
652	} else {
653	$marc_field = MARC::Field->new($tag, $field)
654	}
655	$record->append_fields($marc_field);
656	}
657	;
658	return $record;
659	}
660
565	=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)	661	=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
566		662
567	my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)	663	my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)

Lines 427-429 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+9 lines)
427	choices:	427	choices:
428	Zebra: Zebra	428	Zebra: Zebra
429	Elasticsearch: Elasticsearch	429	Elasticsearch: Elasticsearch
		430	-
		431	- "Elasticsearch MARC format: "
		432	- pref: ElasticsearchMARCFormat
		433	default: "ISO2709"
		434	choices:
		435	"ISO2709": "ISO2709 (exchange format)"
		436	"ARRAY": "Searchable array"
		437	- <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
		438	- <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.

Lines 17-23 Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +81 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19		19
20	use Test::More tests => 4;	20	use Test::More tests => 5;
21	use Test::Exception;	21	use Test::Exception;
22		22
23	use t::lib::Mocks;	23	use t::lib::Mocks;
Lines 120-125 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
120	plan tests => 49;	120	plan tests => 49;
121		121
122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
		123	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
123		124
124	my @mappings = (	125	my @mappings = (
125	{	126	{
Lines 491-493 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
491	ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");	492	ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
492	ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");	493	ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
493	};	494	};
494	-	495
		496	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
		497
		498	plan tests => 6;
		499
		500	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
		501	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
		502
		503	my @mappings = (
		504	{
		505	name => 'control_number',
		506	type => 'string',
		507	facet => 0,
		508	suggestible => 0,
		509	sort => undef,
		510	marc_type => 'marc21',
		511	marc_field => '001',
		512	}
		513	);
		514
		515	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
		516	$se->mock('_foreach_mapping', sub {
		517	my ($self, $sub) = @_;
		518
		519	foreach my $map (@mappings) {
		520	$sub->(
		521	$map->{name},
		522	$map->{type},
		523	$map->{facet},
		524	$map->{suggestible},
		525	$map->{sort},
		526	$map->{marc_type},
		527	$map->{marc_field}
		528	);
		529	}
		530	});
		531
		532	my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
		533
		534	my $marc_record_1 = MARC::Record->new();
		535	$marc_record_1->leader(' cam 22 a 4500');
		536	$marc_record_1->append_fields(
		537	MARC::Field->new('001', '123'),
		538	MARC::Field->new('020', '', '', a => '1-56619-909-3'),
		539	MARC::Field->new('100', '', '', a => 'Author 1'),
		540	MARC::Field->new('110', '', '', a => 'Corp Author'),
		541	MARC::Field->new('210', '', '', a => 'Title 1'),
		542	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
		543	MARC::Field->new('999', '', '', c => '1234567'),
		544	);
		545	my $marc_record_2 = MARC::Record->new();
		546	$marc_record_2->leader(' cam 22 a 4500');
		547	$marc_record_2->append_fields(
		548	MARC::Field->new('100', '', '', a => 'Author 2'),
		549	# MARC::Field->new('210', '', '', a => 'Title 2'),
		550	# MARC::Field->new('245', '', '', a => 'Title: second record'),
		551	MARC::Field->new('999', '', '', c => '1234568'),
		552	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
		553	);
		554	my $records = [$marc_record_1, $marc_record_2];
		555
		556	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
		557
		558	my $docs = $see->marc_records_to_documents($records);
		559
560	# First record:
561	is(scalar @{$docs}, 2, 'Two records converted to documents');
562
563	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
564
565	is_deeply($docs->[0][1]->{control_number}, ['123'], 'First record control number should be set correctly');
566
567	is($docs->[0][1]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
568
569	my $decoded_marc_record = $see->decode_record_from_result($docs->[0][1]);
570
571	ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
572	is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
573	};