View | Details | Raw Unified | Return to bug 22258
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-18 / +114 lines)
Lines 409-414 sub marc_records_to_documents { Link Here
409
    my $control_fields_rules = $rules->{control_fields};
409
    my $control_fields_rules = $rules->{control_fields};
410
    my $data_fields_rules = $rules->{data_fields};
410
    my $data_fields_rules = $rules->{data_fields};
411
    my $marcflavour = lc C4::Context->preference('marcflavour');
411
    my $marcflavour = lc C4::Context->preference('marcflavour');
412
    my $use_array = C4::Context->preference('ElasticsearchMARCFormat') eq 'ARRAY';
412
413
413
    my @record_documents;
414
    my @record_documents;
414
415
Lines 514-539 sub marc_records_to_documents { Link Here
514
515
515
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
516
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
516
        $record->encoding('UTF-8');
517
        $record->encoding('UTF-8');
517
        my @warnings;
518
        if ($use_array) {
518
        {
519
            $record_document->{'marc_data_array'} = $self->_marc_to_array($record);
519
            # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
520
            $record_document->{'marc_format'} = 'ARRAY';
520
            local $SIG{__WARN__} = sub {
521
        } else {
521
                push @warnings, $_[0];
522
            my @warnings;
522
            };
523
            {
523
            $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
524
                # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
524
        }
525
                local $SIG{__WARN__} = sub {
525
        if (@warnings) {
526
                    push @warnings, $_[0];
526
            # Suppress warnings if record length exceeded
527
                };
527
            unless (substr($record->leader(), 0, 5) eq '99999') {
528
                $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
528
                foreach my $warning (@warnings) {
529
            }
529
                    carp $warning;
530
            if (@warnings) {
531
                # Suppress warnings if record length exceeded
532
                unless (substr($record->leader(), 0, 5) eq '99999') {
533
                    foreach my $warning (@warnings) {
534
                        carp $warning;
535
                    }
530
                }
536
                }
537
                $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
538
                $record_document->{'marc_format'} = 'MARCXML';
539
            }
540
            else {
541
                $record_document->{'marc_format'} = 'base64ISO2709';
531
            }
542
            }
532
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
533
            $record_document->{'marc_format'} = 'MARCXML';
534
        }
535
        else {
536
            $record_document->{'marc_format'} = 'base64ISO2709';
537
        }
543
        }
538
        my $id = $record->subfield('999', 'c');
544
        my $id = $record->subfield('999', 'c');
539
        push @record_documents, [$id, $record_document];
545
        push @record_documents, [$id, $record_document];
Lines 541-546 sub marc_records_to_documents { Link Here
541
    return \@record_documents;
547
    return \@record_documents;
542
}
548
}
543
549
550
=head2 _marc_to_array($record)
551
552
    my @fields = _marc_to_array($record)
553
554
Convert a MARC::Record to an array modeled after MARC-in-JSON
555
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
556
557
=over 4
558
559
=item C<$record>
560
561
A MARC::Record object
562
563
=back
564
565
=cut
566
567
sub _marc_to_array {
568
    my ($self, $record) = @_;
569
570
    my $data = {
571
        leader => $record->leader(),
572
        fields => []
573
    };
574
    for my $field ($record->fields()) {
575
        my $tag = $field->tag();
576
        if ($field->is_control_field()) {
577
            push @{$data->{fields}}, {$tag => $field->data()};
578
        } else {
579
            my $subfields = ();
580
            foreach my $subfield ($field->subfields()) {
581
                my ($code, $contents) = @{$subfield};
582
                push @{$subfields}, {$code => $contents};
583
            }
584
            push @{$data->{fields}}, {
585
                $tag => {
586
                    ind1 => $field->indicator(1),
587
                    ind2 => $field->indicator(2),
588
                    subfields => $subfields
589
                }
590
            };
591
        }
592
    }
593
    return $data;
594
}
595
596
=head2 _array_to_marc($data)
597
598
    my $record = _array_to_marc($data)
599
600
Convert an array modeled after MARC-in-JSON to a MARC::Record
601
602
=over 4
603
604
=item C<$data>
605
606
An array modeled after MARC-in-JSON
607
(see https://github.com/marc4j/marc4j/wiki/MARC-in-JSON-Description)
608
609
=back
610
611
=cut
612
613
sub _array_to_marc {
614
    my ($self, $data) = @_;
615
616
    my $record = MARC::Record->new();
617
618
    $record->leader($data->{leader});
619
    for my $field (@{$data->{fields}}) {
620
        my $tag = (keys %{$field})[0];
621
        $field = $field->{$tag};
622
        my $marc_field;
623
        if (ref($field) eq 'HASH') {
624
            my @subfields;
625
            foreach my $subfield (@{$field->{subfields}}) {
626
                my $code = (keys %{$subfield})[0];
627
                push @subfields, $code;
628
                push @subfields, $subfield->{$code};
629
            }
630
            $marc_field = MARC::Field->new($tag, $field->{ind1}, $field->{ind2}, @subfields);
631
        } else {
632
            $marc_field = MARC::Field->new($tag, $field)
633
        }
634
        $record->append_fields($marc_field);
635
    }
636
;
637
    return $record;
638
}
639
544
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
640
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
545
641
546
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
642
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+3 lines)
Lines 383-388 sub decode_record_from_result { Link Here
383
    elsif ($result->{marc_format} eq 'MARCXML') {
383
    elsif ($result->{marc_format} eq 'MARCXML') {
384
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
384
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
385
    }
385
    }
386
    elsif ($result->{marc_format} eq 'ARRAY') {
387
        return $self->_array_to_marc($result->{marc_data_array});
388
    }
386
    else {
389
    else {
387
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
390
        Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
388
    }
391
    }
(-)a/admin/searchengine/elasticsearch/field_config.yaml (+3 lines)
Lines 10-15 general: Link Here
10
      type: text
10
      type: text
11
      analyzer: keyword
11
      analyzer: keyword
12
      index: false
12
      index: false
13
    marc_data_array:
14
      type: object
15
      dynamic: true
13
    marc_format:
16
    marc_format:
14
      store: true
17
      store: true
15
      type: text
18
      type: text
(-)a/installer/data/mysql/atomicupdate/bug_22258.perl (+11 lines)
Line 0 Link Here
1
$DBversion = 'XXX'; # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
    $dbh->do(q{
4
        INSERT IGNORE INTO `systempreferences` (`variable`,`value`,`explanation`,`options`,`type`) VALUES
5
        ('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice')
6
    });
7
8
    # Always end with this (adjust the bug info)
9
    SetVersion( $DBversion );
10
    print "Upgrade to $DBversion done (Bug 22258 - Add ElasticsearchMARCFormat preference)\n";
11
}
(-)a/installer/data/mysql/sysprefs.sql (+1 lines)
Lines 157-162 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
157
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
157
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
158
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
158
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
159
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
159
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
160
('ElasticsearchMARCFormat', 'ISO2709', 'ISO2709|ARRAY', 'Elasticsearch MARC format. ISO2709 format is recommended as it is faster and takes less space, whereas array is searchable.', 'Choice'),
160
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
161
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
161
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
162
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
162
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
163
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+9 lines)
Lines 427-429 Administration: Link Here
427
              choices:
427
              choices:
428
                Zebra: Zebra
428
                Zebra: Zebra
429
                Elasticsearch: Elasticsearch
429
                Elasticsearch: Elasticsearch
430
        -
431
            - "Elasticsearch MARC format: "
432
            - pref: ElasticsearchMARCFormat
433
              default: "ISO2709"
434
              choices:
435
                "ISO2709": "ISO2709 (exchange format)"
436
                "ARRAY": "Searchable array"
437
            - <br>ISO2709 format is recommended as it is faster and takes less space, whereas array format makes the full MARC record searchable.
438
            - <br><strong>NOTE:</strong> Making the full record searchable may have a negative effect on relevance ranking of search results.
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +81 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 4;
20
use Test::More tests => 5;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
Lines 120-125 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
120
    plan tests => 47;
120
    plan tests => 47;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
123
124
124
    my @mappings = (
125
    my @mappings = (
125
        {
126
        {
Lines 465-467 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
465
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
466
    ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
466
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
467
    ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
467
};
468
};
468
- 
469
470
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
471
472
    plan tests => 6;
473
474
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
475
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
476
477
    my @mappings = (
478
        {
479
            name => 'control_number',
480
            type => 'string',
481
            facet => 0,
482
            suggestible => 0,
483
            sort => undef,
484
            marc_type => 'marc21',
485
            marc_field => '001',
486
        }
487
    );
488
489
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
490
    $se->mock('_foreach_mapping', sub {
491
        my ($self, $sub) = @_;
492
493
        foreach my $map (@mappings) {
494
            $sub->(
495
                $map->{name},
496
                $map->{type},
497
                $map->{facet},
498
                $map->{suggestible},
499
                $map->{sort},
500
                $map->{marc_type},
501
                $map->{marc_field}
502
            );
503
        }
504
    });
505
506
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
507
508
    my $marc_record_1 = MARC::Record->new();
509
    $marc_record_1->leader('     cam  22      a 4500');
510
    $marc_record_1->append_fields(
511
        MARC::Field->new('001', '123'),
512
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
513
        MARC::Field->new('100', '', '', a => 'Author 1'),
514
        MARC::Field->new('110', '', '', a => 'Corp Author'),
515
        MARC::Field->new('210', '', '', a => 'Title 1'),
516
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
517
        MARC::Field->new('999', '', '', c => '1234567'),
518
    );
519
    my $marc_record_2 = MARC::Record->new();
520
    $marc_record_2->leader('     cam  22      a 4500');
521
    $marc_record_2->append_fields(
522
        MARC::Field->new('100', '', '', a => 'Author 2'),
523
        # MARC::Field->new('210', '', '', a => 'Title 2'),
524
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
525
        MARC::Field->new('999', '', '', c => '1234568'),
526
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
527
    );
528
    my $records = [$marc_record_1, $marc_record_2];
529
530
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
531
532
    my $docs = $see->marc_records_to_documents($records);
533
534
    # First record:
535
    is(scalar @{$docs}, 2, 'Two records converted to documents');
536
537
    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
538
539
    is_deeply($docs->[0][1]->{control_number}, ['123'], 'First record control number should be set correctly');
540
541
    is($docs->[0][1]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
542
543
    my $decoded_marc_record = $see->decode_record_from_result($docs->[0][1]);
544
545
    ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
546
    is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
547
};

Return to bug 22258