View | Details | Raw Unified | Return to bug 22258
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-18 / +114 lines)
Lines 416-421 sub marc_records_to_documents { Link Here
416
    my $control_fields_rules = $rules->{control_fields};
416
    my $control_fields_rules = $rules->{control_fields};
417
    my $data_fields_rules = $rules->{data_fields};
417
    my $data_fields_rules = $rules->{data_fields};
418
    my $marcflavour = lc C4::Context->preference('marcflavour');
418
    my $marcflavour = lc C4::Context->preference('marcflavour');
419
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
419
420
420
    my @record_documents;
421
    my @record_documents;
421
422
Lines 535-560 sub marc_records_to_documents { Link Here
535
536
536
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
537
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
537
        $record->encoding('UTF-8');
538
        $record->encoding('UTF-8');
538
        my @warnings;
539
        if ($use_array) {
539
        {
540
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
540
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
541
            $record_document->{'marc_format'} = 'ARRAY';
541
            local $SIG{__WARN__} = sub {
542
        } else {
542
                push @warnings, $_[0];
543
            my @warnings;
543
            };
544
            {
544
            $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
545
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
545
        }
546
                local $SIG{__WARN__} = sub {
546
        if (@warnings) {
547
                    push @warnings, $_[0];
547
            # Suppress warnings if record length exceeded
548
                };
548
            unless (substr($record->leader(), 0, 5) eq '99999') {
549
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
549
                foreach my $warning (@warnings) {
550
            }
550
                    carp $warning;
551
            if (@warnings) {
552
                # Suppress warnings if record length exceeded
553
                unless (substr($record->leader(), 0, 5) eq '99999') {
554
                    foreach my $warning (@warnings) {
555
                        carp $warning;
556
                    }
551
                }
557
                }
558
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
559
                $record_document->{'marc_format'} = 'MARCXML';
560
            }
561
            else {
562
                $record_document->{'marc_format'} = 'base64ISO2709';
552
            }
563
            }
553
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
554
            $record_document->{'marc_format'} = 'MARCXML';
555
        }
556
        else {
557
            $record_document->{'marc_format'} = 'base64ISO2709';
558
        }
564
        }
559
        my $id = $record->subfield('999', 'c');
565
        my $id = $record->subfield('999', 'c');
560
        push @record_documents, [$id, $record_document];
566
        push @record_documents, [$id, $record_document];
Lines 562-567 sub marc_records_to_documents { Link Here
562
    return \@record_documents;
568
    return \@record_documents;
563
}
569
}
564
570
571
=head2 _marc_to_array($record)
572
573
    my @fields = _marc_to_array($record)
574
575
Convert a MARC::Record to an array modeled after MARC-in-JSON
576
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
577
578
=over 4
579
580
=item C<$record>
581
582
A MARC::Record object
583
584
=back
585
586
=cut
587
588
sub _marc_to_array {
589
    my ($self, $record) = @_;
590
591
    my $data = {
592
        leader => $record->leader(),
593
        fields => []
594
    };
595
    for my $field ($record->fields()) {
596
        my $tag = $field->tag();
597
        if ($field->is_control_field()) {
598
            push @{$data->{fields}}, {$tag => $field->data()};
599
        } else {
600
            my $subfields = ();
601
            foreach my $subfield ($field->subfields()) {
602
                my ($code, $contents) = @{$subfield};
603
                push @{$subfields}, {$code => $contents};
604
            }
605
            push @{$data->{fields}}, {
606
                $tag => {
607
                    ind1 => $field->indicator(1),
608
                    ind2 => $field->indicator(2),
609
                    subfields => $subfields
610
                }
611
            };
612
        }
613
    }
614
    return $data;
615
}
616
617
=head2 _array_to_marc($data)
618
619
    my $record = _array_to_marc($data)
620
621
Convert an array modeled after MARC-in-JSON to a MARC::Record
622
623
=over 4
624
625
=item C<$data>
626
627
An array modeled after MARC-in-JSON
628
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
629
630
=back
631
632
=cut
633
634
sub _array_to_marc {
635
    my ($self, $data) = @_;
636
637
    my $record = MARC::Record->new();
638
639
    $record->leader($data->{leader});
640
    for my $field (@{$data->{fields}}) {
641
        my $tag = (keys %{$field})[0];
642
        $field = $field->{$tag};
643
        my $marc_field;
644
        if (ref($field) eq 'HASH') {
645
            my @subfields;
646
            foreach my $subfield (@{$field->{subfields}}) {
647
                my $code = (keys %{$subfield})[0];
648
                push @subfields, $code;
649
                push @subfields, $subfield->{$code};
650
            }
651
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
652
        } else {
653
            $marc_field = MARC::Field->new($tag, $field)
654
        }
655
        $record->append_fields($marc_field);
656
    }
657
;
658
    return $record;
659
}
660
565
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
661
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
566
662
567
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
663
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+3 lines)
Lines 383-388 sub decode_record_from_result { Link Here
383
    elsif ($result->{marc_format} eq 'MARCXML') {
383
    elsif ($result->{marc_format} eq 'MARCXML') {
384
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
384
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
385
    }
385
    }
386
    elsif ($result->{marc_format} eq 'ARRAY') {
387
        return $self->_array_to_marc($result->{marc_data_array});
388
    }
386
    else {
389
    else {
387
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
390
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
388
    }
391
    }
(-)a/admin/searchengine/elasticsearch/field_config.yaml (+3 lines)
Lines 10-15 general: Link Here
10
      type: text
10
      type: text
11
      analyzer: keyword
11
      analyzer: keyword
12
      index: false
12
      index: false
13
    marc_data_array:
14
      type: object
15
      dynamic: true
13
    marc_format:
16
    marc_format:
14
      store: true
17
      store: true
15
      type: text
18
      type: text
(-)a/installer/data/mysql/atomicupdate/bug_22258.perl (+11 lines)
Line 0 Link Here
1
$DBversion = 'XXX'; # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
    $dbh->do(q{
4
        INSERT IGNORE INTO `systempreferences` (`variable`,`value`,`explanation`,`options`,`type`) VALUES
5
        ('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice')
6
    });
7
8
    # Always end with this (adjust the bug info)
9
    SetVersion( $DBversion );
10
    print "Upgrade to $DBversion done (Bug 22258 - Add ElasticsearchMARCFormat preference)\n";
11
}
(-)a/installer/data/mysql/sysprefs.sql (+1 lines)
Lines 157-162 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
157
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
157
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
158
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
158
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
159
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
159
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
160
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
160
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
161
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
161
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
162
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
162
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
163
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+9 lines)
Lines 427-429 Administration: Link Here
427
              choices:
427
              choices:
428
                Zebra: Zebra
428
                Zebra: Zebra
429
                Elasticsearch: Elasticsearch
429
                Elasticsearch: Elasticsearch
430
        -
431
            - "Elasticsearch MARC format: "
432
            - pref: ElasticsearchMARCFormat
433
              default: "ISO2709"
434
              choices:
435
                "ISO2709": "ISO2709 (exchange format)"
436
                "ARRAY": "Searchable array"
437
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
438
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +81 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 4;
20
use Test::More tests => 5;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
Lines 120-125 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
120
    plan tests => 49;
120
    plan tests => 49;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
123
124
124
    my @mappings = (
125
    my @mappings = (
125
        {
126
        {
Lines 491-493 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
491
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
492
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
492
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
493
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
493
};
494
};
494
- 
495
496
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
497
498
    plan tests => 6;
499
500
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
501
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
502
503
    my @mappings = (
504
        {
505
            name => 'control_number',
506
            type => 'string',
507
            facet => 0,
508
            suggestible => 0,
509
            sort => undef,
510
            marc_type => 'marc21',
511
            marc_field => '001',
512
        }
513
    );
514
515
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
516
    $se->mock('_foreach_mapping', sub {
517
        my ($self, $sub) = @_;
518
519
        foreach my $map (@mappings) {
520
            $sub->(
521
                $map->{name},
522
                $map->{type},
523
                $map->{facet},
524
                $map->{suggestible},
525
                $map->{sort},
526
                $map->{marc_type},
527
                $map->{marc_field}
528
            );
529
        }
530
    });
531
532
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
533
534
    my $marc_record_1 = MARC::Record->new();
535
    $marc_record_1->leader('     cam  22      a 4500');
536
    $marc_record_1->append_fields(
537
        MARC::Field->new('001', '123'),
538
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
539
        MARC::Field->new('100', '', '', a => 'Author 1'),
540
        MARC::Field->new('110', '', '', a => 'Corp Author'),
541
        MARC::Field->new('210', '', '', a => 'Title 1'),
542
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
543
        MARC::Field->new('999', '', '', c => '1234567'),
544
    );
545
    my $marc_record_2 = MARC::Record->new();
546
    $marc_record_2->leader('     cam  22      a 4500');
547
    $marc_record_2->append_fields(
548
        MARC::Field->new('100', '', '', a => 'Author 2'),
549
        # MARC::Field->new('210', '', '', a => 'Title 2'),
550
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
551
        MARC::Field->new('999', '', '', c => '1234568'),
552
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
553
    );
554
    my $records = [$marc_record_1, $marc_record_2];
555
556
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
557
558
    my $docs = $see->marc_records_to_documents($records);
559
560
    # First record:
561
    is(scalar @{$docs}, 2, 'Two records converted to documents');
562
563
    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
564
565
    is_deeply($docs->[0][1]->{control_number}, ['123'], 'First record control number should be set correctly');
566
567
    is($docs->[0][1]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
568
569
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0][1]);
570
571
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
572
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
573
};

Return to bug 22258