View | Details | Raw Unified | Return to bug 22258
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-18 / +114 lines)
Lines 408-413 sub marc_records_to_documents { Link Here
408
    my $control_fields_rules = $rules->{control_fields};
408
    my $control_fields_rules = $rules->{control_fields};
409
    my $data_fields_rules = $rules->{data_fields};
409
    my $data_fields_rules = $rules->{data_fields};
410
    my $marcflavour = lc C4::Context->preference('marcflavour');
410
    my $marcflavour = lc C4::Context->preference('marcflavour');
411
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
411
412
412
    my @record_documents;
413
    my @record_documents;
413
414
Lines 527-552 sub marc_records_to_documents { Link Here
527
528
528
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
529
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
529
        $record->encoding('UTF-8');
530
        $record->encoding('UTF-8');
530
        my @warnings;
531
        if ($use_array) {
531
        {
532
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
532
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
533
            $record_document->{'marc_format'} = 'ARRAY';
533
            local $SIG{__WARN__} = sub {
534
        } else {
534
                push @warnings, $_[0];
535
            my @warnings;
535
            };
536
            {
536
            $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
537
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
537
        }
538
                local $SIG{__WARN__} = sub {
538
        if (@warnings) {
539
                    push @warnings, $_[0];
539
            # Suppress warnings if record length exceeded
540
                };
540
            unless (substr($record->leader(), 0, 5) eq '99999') {
541
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
541
                foreach my $warning (@warnings) {
542
            }
542
                    carp $warning;
543
            if (@warnings) {
544
                # Suppress warnings if record length exceeded
545
                unless (substr($record->leader(), 0, 5) eq '99999') {
546
                    foreach my $warning (@warnings) {
547
                        carp $warning;
548
                    }
543
                }
549
                }
550
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
551
                $record_document->{'marc_format'} = 'MARCXML';
552
            }
553
            else {
554
                $record_document->{'marc_format'} = 'base64ISO2709';
544
            }
555
            }
545
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
546
            $record_document->{'marc_format'} = 'MARCXML';
547
        }
548
        else {
549
            $record_document->{'marc_format'} = 'base64ISO2709';
550
        }
556
        }
551
        my $id = $record->subfield('999', 'c');
557
        my $id = $record->subfield('999', 'c');
552
        push @record_documents, [$id, $record_document];
558
        push @record_documents, [$id, $record_document];
Lines 554-559 sub marc_records_to_documents { Link Here
554
    return \@record_documents;
560
    return \@record_documents;
555
}
561
}
556
562
563
=head2 _marc_to_array($record)
564
565
    my @fields = _marc_to_array($record)
566
567
Convert a MARC::Record to an array modeled after MARC-in-JSON
568
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
569
570
=over 4
571
572
=item C<$record>
573
574
A MARC::Record object
575
576
=back
577
578
=cut
579
580
sub _marc_to_array {
581
    my ($self, $record) = @_;
582
583
    my $data = {
584
        leader => $record->leader(),
585
        fields => []
586
    };
587
    for my $field ($record->fields()) {
588
        my $tag = $field->tag();
589
        if ($field->is_control_field()) {
590
            push @{$data->{fields}}, {$tag => $field->data()};
591
        } else {
592
            my $subfields = ();
593
            foreach my $subfield ($field->subfields()) {
594
                my ($code, $contents) = @{$subfield};
595
                push @{$subfields}, {$code => $contents};
596
            }
597
            push @{$data->{fields}}, {
598
                $tag => {
599
                    ind1 => $field->indicator(1),
600
                    ind2 => $field->indicator(2),
601
                    subfields => $subfields
602
                }
603
            };
604
        }
605
    }
606
    return $data;
607
}
608
609
=head2 _array_to_marc($data)
610
611
    my $record = _array_to_marc($data)
612
613
Convert an array modeled after MARC-in-JSON to a MARC::Record
614
615
=over 4
616
617
=item C<$data>
618
619
An array modeled after MARC-in-JSON
620
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
621
622
=back
623
624
=cut
625
626
sub _array_to_marc {
627
    my ($self, $data) = @_;
628
629
    my $record = MARC::Record->new();
630
631
    $record->leader($data->{leader});
632
    for my $field (@{$data->{fields}}) {
633
        my $tag = (keys %{$field})[0];
634
        $field = $field->{$tag};
635
        my $marc_field;
636
        if (ref($field) eq 'HASH') {
637
            my @subfields;
638
            foreach my $subfield (@{$field->{subfields}}) {
639
                my $code = (keys %{$subfield})[0];
640
                push @subfields, $code;
641
                push @subfields, $subfield->{$code};
642
            }
643
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
644
        } else {
645
            $marc_field = MARC::Field->new($tag, $field)
646
        }
647
        $record->append_fields($marc_field);
648
    }
649
;
650
    return $record;
651
}
652
557
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
653
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
558
654
559
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
655
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+3 lines)
Lines 382-387 sub decode_record_from_result { Link Here
382
    elsif ($result->{marc_format} eq 'MARCXML') {
382
    elsif ($result->{marc_format} eq 'MARCXML') {
383
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
383
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
384
    }
384
    }
385
    elsif ($result->{marc_format} eq 'ARRAY') {
386
        return $self->_array_to_marc($result->{marc_data_array});
387
    }
385
    else {
388
    else {
386
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
389
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
387
    }
390
    }
(-)a/admin/searchengine/elasticsearch/field_config.yaml (+3 lines)
Lines 10-15 general: Link Here
10
      type: text
10
      type: text
11
      analyzer: keyword
11
      analyzer: keyword
12
      index: false
12
      index: false
13
    marc_data_array:
14
      type: object
15
      dynamic: true
13
    marc_format:
16
    marc_format:
14
      store: true
17
      store: true
15
      type: text
18
      type: text
(-)a/installer/data/mysql/atomicupdate/bug_22258.perl (+11 lines)
Line 0 Link Here
1
$DBversion = 'XXX'; # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
    $dbh->do(q{
4
        INSERT IGNORE INTO `systempreferences` (`variable`,`value`,`explanation`,`options`,`type`) VALUES
5
        ('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice')
6
    });
7
8
    # Always end with this (adjust the bug info)
9
    SetVersion( $DBversion );
10
    print "Upgrade to $DBversion done (Bug 22258 - Add ElasticsearchMARCFormat preference)\n";
11
}
(-)a/installer/data/mysql/sysprefs.sql (+1 lines)
Lines 159-164 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
159
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
159
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
160
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
160
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
161
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
161
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
162
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
162
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
163
('EmailAddressForSuggestions','','',' If you choose EmailAddressForSuggestions you have to enter a valid email address: ','free'),
163
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
164
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
164
('EmailPurchaseSuggestions','0','0|EmailAddressForSuggestions|BranchEmailAddress|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'),
165
('EmailPurchaseSuggestions','0','0|EmailAddressForSuggestions|BranchEmailAddress|KohaAdminEmailAddress','Choose email address that new purchase suggestions will be sent to: ','Choice'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+9 lines)
Lines 439-441 Administration: Link Here
439
              choices:
439
              choices:
440
                Zebra: Zebra
440
                Zebra: Zebra
441
                Elasticsearch: Elasticsearch
441
                Elasticsearch: Elasticsearch
442
        -
443
            - "Elasticsearch MARC format: "
444
            - pref: ElasticsearchMARCFormat
445
              default: "ISO2709"
446
              choices:
447
                "ISO2709": "ISO2709 (exchange format)"
448
                "ARRAY": "Searchable array"
449
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
450
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +81 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 4;
20
use Test::More tests => 5;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
Lines 120-125 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
120
    plan tests => 50;
120
    plan tests => 50;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
123
124
124
    my @mappings = (
125
    my @mappings = (
125
        {
126
        {
Lines 503-505 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
503
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
504
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
504
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
505
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
505
};
506
};
506
- 
507
508
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
509
510
    plan tests => 6;
511
512
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
513
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
514
515
    my @mappings = (
516
        {
517
            name => 'control_number',
518
            type => 'string',
519
            facet => 0,
520
            suggestible => 0,
521
            sort => undef,
522
            marc_type => 'marc21',
523
            marc_field => '001',
524
        }
525
    );
526
527
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
528
    $se->mock('_foreach_mapping', sub {
529
        my ($self, $sub) = @_;
530
531
        foreach my $map (@mappings) {
532
            $sub->(
533
                $map->{name},
534
                $map->{type},
535
                $map->{facet},
536
                $map->{suggestible},
537
                $map->{sort},
538
                $map->{marc_type},
539
                $map->{marc_field}
540
            );
541
        }
542
    });
543
544
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
545
546
    my $marc_record_1 = MARC::Record->new();
547
    $marc_record_1->leader('     cam  22      a 4500');
548
    $marc_record_1->append_fields(
549
        MARC::Field->new('001', '123'),
550
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
551
        MARC::Field->new('100', '', '', a => 'Author 1'),
552
        MARC::Field->new('110', '', '', a => 'Corp Author'),
553
        MARC::Field->new('210', '', '', a => 'Title 1'),
554
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
555
        MARC::Field->new('999', '', '', c => '1234567'),
556
    );
557
    my $marc_record_2 = MARC::Record->new();
558
    $marc_record_2->leader('     cam  22      a 4500');
559
    $marc_record_2->append_fields(
560
        MARC::Field->new('100', '', '', a => 'Author 2'),
561
        # MARC::Field->new('210', '', '', a => 'Title 2'),
562
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
563
        MARC::Field->new('999', '', '', c => '1234568'),
564
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
565
    );
566
    my $records = [$marc_record_1, $marc_record_2];
567
568
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
569
570
    my $docs = $see->marc_records_to_documents($records);
571
572
    # First record:
573
    is(scalar @{$docs}, 2, 'Two records converted to documents');
574
575
    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
576
577
    is_deeply($docs->[0][1]->{control_number}, ['123'], 'First record control number should be set correctly');
578
579
    is($docs->[0][1]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
580
581
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0][1]);
582
583
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
584
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
585
};

Return to bug 22258