View | Details | Raw Unified | Return to bug 25273
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-1 / +1 lines)
Lines 629-635 sub AddAuthority { Link Here
629
    $record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
629
    $record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
630
    # Update
630
    # Update
631
    $dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
631
    $dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
632
    ModZebra( $authid, 'specialUpdate', 'authorityserver', $record );
632
    ModZebra( $authid, 'specialUpdate', 'authorityserver', { record => $record, authtypecode => $authtypecode } );
633
633
634
    return ( $authid );
634
    return ( $authid );
635
}
635
}
(-)a/C4/Biblio.pm (-5 / +6 lines)
Lines 2546-2553 $op is specialUpdate or recordDelete, and is used to know what we want to do Link Here
2546
2546
2547
$server is the server that we want to update
2547
$server is the server that we want to update
2548
2548
2549
$record is the update MARC record if it's available. If it's not supplied
2549
$record is a hash including the authtypecode and updated MARC record. This is only used for
2550
and is needed, it'll be loaded from the database.
2550
Elasticsearch authorities. If it's not supplied a biblio will be loaded from the database.
2551
2551
2552
=cut
2552
=cut
2553
2553
Lines 2567-2579 sub ModZebra { Link Here
2567
            }
2567
            }
2568
        );
2568
        );
2569
        if ( $op eq 'specialUpdate' ) {
2569
        if ( $op eq 'specialUpdate' ) {
2570
            unless ($record) {
2570
            if ($record) {
2571
                $indexer->update_index_background( [$biblionumber], [$record] );
2572
            } else {
2571
                $record = GetMarcBiblio({
2573
                $record = GetMarcBiblio({
2572
                    biblionumber => $biblionumber,
2574
                    biblionumber => $biblionumber,
2573
                    embed_items  => 1 });
2575
                    embed_items  => 1 });
2576
                $indexer->update_index_background( [$biblionumber], [{ record => $record }] );
2574
            }
2577
            }
2575
            my $records = [$record];
2576
            $indexer->update_index_background( [$biblionumber], [$record] );
2577
        }
2578
        }
2578
        elsif ( $op eq 'recordDelete' ) {
2579
        elsif ( $op eq 'recordDelete' ) {
2579
            $indexer->delete_index_background( [$biblionumber] );
2580
            $indexer->delete_index_background( [$biblionumber] );
(-)a/Koha/SearchEngine/Elasticsearch.pm (-15 / +16 lines)
Lines 74-79 sub new { Link Here
74
    my $self = $class->SUPER::new(@_);
74
    my $self = $class->SUPER::new(@_);
75
    # Check for a valid index
75
    # Check for a valid index
76
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index;
76
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index;
77
77
    return $self;
78
    return $self;
78
}
79
}
79
80
Lines 237-246 sub get_elasticsearch_mappings { Link Here
237
                }
238
                }
238
            }
239
            }
239
        );
240
        );
241
        $mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
240
        $all_mappings{$self->index} = $mappings;
242
        $all_mappings{$self->index} = $mappings;
241
    }
243
    }
242
    $self->sort_fields(\%{$sort_fields{$self->index}});
244
    $self->sort_fields(\%{$sort_fields{$self->index}});
243
244
    return $all_mappings{$self->index};
245
    return $all_mappings{$self->index};
245
}
246
}
246
247
Lines 524-531 sub marc_records_to_documents { Link Here
524
525
525
    my @record_documents;
526
    my @record_documents;
526
527
528
    my %auth_match_headings;
529
    if( $self->index eq 'authorities' ){
530
        my @auth_types = Koha::Authority::Types->search();
531
        %auth_match_headings = map { $_->authtypecode => $_->auth_tag_to_report } @auth_types;
532
    }
533
527
    foreach my $record (@{$records}) {
534
    foreach my $record (@{$records}) {
528
        my $record_document = {};
535
        my $record_document = {};
536
537
        if ( defined $record->{authtypecode} ){
538
            my $field = $record->{record}->field( $auth_match_headings{ $record->{authtypecode} } );
539
            my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
540
            push @{$record_document->{'match-heading'}}, $heading->search_form if $heading;
541
        }
542
        $record = $record->{record};
543
529
        my $mappings = $rules->{leader};
544
        my $mappings = $rules->{leader};
530
        if ($mappings) {
545
        if ($mappings) {
531
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
546
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
Lines 577-589 sub marc_records_to_documents { Link Here
577
                                }
592
                                }
578
                            );
593
                            );
579
                        }
594
                        }
580
                        if ( @{$mappings} && grep { $_->[0] eq 'match-heading'} @{$mappings} ){
581
                            # Used by the authority linker the match-heading field requires a specific syntax
582
                            # that is specified in C4/Heading
583
                            my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
584
                            next unless $heading;
585
                            push @{$record_document->{'match-heading'}}, $heading->search_form;
586
                        }
587
                    }
595
                    }
588
596
589
                    my $subfields_join_mappings = $data_field_rules->{subfields_join};
597
                    my $subfields_join_mappings = $data_field_rules->{subfields_join};
Lines 606-618 sub marc_records_to_documents { Link Here
606
                                    }
614
                                    }
607
                                );
615
                                );
608
                            }
616
                            }
609
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
610
                                # Used by the authority linker the match-heading field requires a specific syntax
611
                                # that is specified in C4/Heading
612
                                my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
613
                                next unless $heading;
614
                                push @{$record_document->{'match-heading'}}, $heading->search_form;
615
                            }
616
                        }
617
                        }
617
                    }
618
                    }
618
                }
619
                }
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +1 lines)
Lines 476-482 sub build_authorities_query_compat { Link Here
476
476
477
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
477
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
478
        push @indexes, $m;
478
        push @indexes, $m;
479
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '');
479
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
480
    }
480
    }
481
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
481
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
482
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
482
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
(-)a/admin/searchengine/elasticsearch/mappings.yaml (-24 lines)
Lines 695-724 authorities: Link Here
695
        sort: ~
695
        sort: ~
696
        suggestible: ''
696
        suggestible: ''
697
    type: ''
697
    type: ''
698
  Match-heading:
699
    label: Match-heading
700
    mappings:
701
      - facet: ''
702
        marc_field: 100(abcdefghjklmnopqrstvxyz)
703
        marc_type: marc21
704
        sort: ~
705
        suggestible: ''
706
      - facet: ''
707
        marc_field: 111(acdefghjklnpqstvxyz)
708
        marc_type: marc21
709
        sort: ~
710
        suggestible: ''
711
      - facet: ''
712
        marc_field: 100(abcdefghjklmnopqrstvxyz)
713
        marc_type: normarc
714
        sort: ~
715
        suggestible: ''
716
      - facet: ''
717
        marc_field: 111(acdefghjklnpqstvxyz)
718
        marc_type: normarc
719
        sort: ~
720
        suggestible: ''
721
    type: ''
722
  Match-heading-see-from:
698
  Match-heading-see-from:
723
    label: Match-heading-see-from
699
    label: Match-heading-see-from
724
    mappings:
700
    mappings:
(-)a/misc/search_tools/rebuild_elasticsearch.pl (-3 / +5 lines)
Lines 253-260 sub _do_reindex { Link Here
253
    my $commit_count = $commit;
253
    my $commit_count = $commit;
254
    my ( @id_buffer, @commit_buffer );
254
    my ( @id_buffer, @commit_buffer );
255
    while ( my $record = $next->() ) {
255
    while ( my $record = $next->() ) {
256
        my $id     = $record->id // $record->authid;
256
        my $id     = $record->id;
257
        my $record = $record->record;
257
        my $record_prepped;
258
        $record_prepped->{authtypecode} = $record->authtypecode if ref $record eq 'Koha::MetadataRecord::Authority';
259
        $record_prepped->{record} = $record->record;
258
        $count++;
260
        $count++;
259
        if ( $verbose == 1 ) {
261
        if ( $verbose == 1 ) {
260
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
262
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
Lines 263-269 sub _do_reindex { Link Here
263
        }
265
        }
264
266
265
        push @id_buffer,     $id;
267
        push @id_buffer,     $id;
266
        push @commit_buffer, $record;
268
        push @commit_buffer, $record_prepped;
267
        if ( !( --$commit_count ) ) {
269
        if ( !( --$commit_count ) ) {
268
            _log( 1, "Committing $commit records...\n" );
270
            _log( 1, "Committing $commit records...\n" );
269
            $indexer->update_index( \@id_buffer, \@commit_buffer );
271
            $indexer->update_index( \@id_buffer, \@commit_buffer );
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-56 / +26 lines)
Lines 21-26 use Test::More tests => 6; Link Here
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
24
use t::lib::TestBuilder;
24
25
25
use Test::MockModule;
26
use Test::MockModule;
26
27
Lines 31-36 use List::Util qw( any ); Link Here
31
use Koha::SearchEngine::Elasticsearch;
32
use Koha::SearchEngine::Elasticsearch;
32
use Koha::SearchEngine::Elasticsearch::Search;
33
use Koha::SearchEngine::Elasticsearch::Search;
33
34
35
my $schema = Koha::Database->new->schema;
36
$schema->storage->txn_begin;
37
34
subtest '_read_configuration() tests' => sub {
38
subtest '_read_configuration() tests' => sub {
35
39
36
    plan tests => 10;
40
    plan tests => 10;
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336
        MARC::Field->new('999', '', '', c => '1234568'),
340
        MARC::Field->new('999', '', '', c => '1234568'),
337
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
341
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
338
    );
342
    );
339
    my $records = [$marc_record_1, $marc_record_2];
343
    my $records = [
344
        { record => $marc_record_1 },
345
        { record => $marc_record_2 },
346
    ];
340
347
341
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
348
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
342
349
Lines 503-509 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
503
        $large_marc_record->append_fields($item_field);
510
        $large_marc_record->append_fields($item_field);
504
    }
511
    }
505
512
506
    $docs = $see->marc_records_to_documents([$large_marc_record]);
513
    $docs = $see->marc_records_to_documents([{ record => $large_marc_record }]);
507
514
508
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
515
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
509
516
Lines 616-622 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
616
        MARC::Field->new('999', '', '', c => '1234568'),
623
        MARC::Field->new('999', '', '', c => '1234568'),
617
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
624
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
618
    );
625
    );
619
    my $records = [$marc_record_1, $marc_record_2];
626
    my $records = [
627
        { record => $marc_record_1 },
628
        { record => $marc_record_2 },
629
    ];
620
630
621
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
631
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
622
632
Lines 642-647 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
642
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
652
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
643
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
653
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
644
654
655
    my $builder = t::lib::TestBuilder->new;
656
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
657
            auth_tag_to_report => '150'
658
        }
659
    });
660
645
    my @mappings = (
661
    my @mappings = (
646
        {
662
        {
647
            name => 'match',
663
            name => 'match',
Lines 652-708 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
652
            sort => 0,
668
            sort => 0,
653
            marc_type => 'marc21',
669
            marc_type => 'marc21',
654
            marc_field => '150(ae)',
670
            marc_field => '150(ae)',
655
        },
671
        }
656
        {
657
            name => 'heading',
658
            type => 'string',
659
            facet => 0,
660
            suggestible => 0,
661
            searchable => 1,
662
            sort => 0,
663
            marc_type => 'marc21',
664
            marc_field => '150a',
665
        },
666
        {
667
            name => 'heading',
668
            type => 'string',
669
            facet => 0,
670
            suggestible => 0,
671
            searchable => 1,
672
            sort => 0,
673
            marc_type => 'marc21',
674
            marc_field => '150(ae)',
675
        },
676
        {
677
            name => 'heading-main',
678
            type => 'string',
679
            facet => 0,
680
            suggestible => 0,
681
            searchable => 1,
682
            sort => 0,
683
            marc_type => 'marc21',
684
            marc_field => '150a',
685
        },
686
        {
687
            name => 'heading',
688
            type => 'string',
689
            facet => 0,
690
            suggestible => 0,
691
            searchable => 1,
692
            sort => 0,
693
            marc_type => 'marc21',
694
            marc_field => '150',
695
        },
696
        {
697
            name => 'match-heading',
698
            type => 'string',
699
            facet => 0,
700
            suggestible => 0,
701
            searchable => 1,
702
            sort => 0,
703
            marc_type => 'marc21',
704
            marc_field => '150',
705
        },
706
    );
672
    );
707
673
708
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
674
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 735-741 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
735
    $marc_record_2->append_fields(
701
    $marc_record_2->append_fields(
736
        MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
702
        MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
737
    );
703
    );
738
    my $records = [$marc_record_1, $marc_record_2];
704
    my $records = [
705
        { record => $marc_record_1, authtypecode=> $auth_type->authtypecode },
706
        { record => $marc_record_2, authtypecode=> $auth_type->authtypecode },
707
    ];
739
708
740
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
709
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
741
710
Lines 752-754 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
752
        "Second record match-heading should contain the correctly formatted heading without wrong subfield"
721
        "Second record match-heading should contain the correctly formatted heading without wrong subfield"
753
    );
722
    );
754
};
723
};
755
- 
724
725
$schema->storage->txn_rollback;

Return to bug 25273