View | Details | Raw Unified | Return to bug 25273
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-1 / +1 lines)
Lines 629-635 sub AddAuthority { Link Here
629
    $record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
629
    $record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
630
    # Update
630
    # Update
631
    $dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
631
    $dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
632
    ModZebra( $authid, 'specialUpdate', 'authorityserver', $record );
632
    ModZebra( $authid, 'specialUpdate', 'authorityserver', { record => $record, authtypecode => $authtypecode } );
633
633
634
    return ( $authid );
634
    return ( $authid );
635
}
635
}
(-)a/C4/Biblio.pm (-5 / +6 lines)
Lines 2546-2553 $op is specialUpdate or recordDelete, and is used to know what we want to do Link Here
2546
2546
2547
$server is the server that we want to update
2547
$server is the server that we want to update
2548
2548
2549
$record is the update MARC record if it's available. If it's not supplied
2549
$record is a hash including the authtypecode and updated MARC record. This is only used for
2550
and is needed, it'll be loaded from the database.
2550
Elasticsearch authorities. If it's not supplied a biblio will be loaded from the database.
2551
2551
2552
=cut
2552
=cut
2553
2553
Lines 2567-2579 sub ModZebra { Link Here
2567
            }
2567
            }
2568
        );
2568
        );
2569
        if ( $op eq 'specialUpdate' ) {
2569
        if ( $op eq 'specialUpdate' ) {
2570
            unless ($record) {
2570
            if ($record) {
2571
                $indexer->update_index_background( [$biblionumber], [$record] );
2572
            } else {
2571
                $record = GetMarcBiblio({
2573
                $record = GetMarcBiblio({
2572
                    biblionumber => $biblionumber,
2574
                    biblionumber => $biblionumber,
2573
                    embed_items  => 1 });
2575
                    embed_items  => 1 });
2576
                $indexer->update_index_background( [$biblionumber], [{ record => $record }] );
2574
            }
2577
            }
2575
            my $records = [$record];
2576
            $indexer->update_index_background( [$biblionumber], [$record] );
2577
        }
2578
        }
2578
        elsif ( $op eq 'recordDelete' ) {
2579
        elsif ( $op eq 'recordDelete' ) {
2579
            $indexer->delete_index_background( [$biblionumber] );
2580
            $indexer->delete_index_background( [$biblionumber] );
(-)a/Koha/SearchEngine/Elasticsearch.pm (-15 / +15 lines)
Lines 234-243 sub get_elasticsearch_mappings { Link Here
234
                }
234
                }
235
            }
235
            }
236
        );
236
        );
237
        $mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
237
        $all_mappings{$self->index} = $mappings;
238
        $all_mappings{$self->index} = $mappings;
238
    }
239
    }
239
    $self->sort_fields(\%{$sort_fields{$self->index}});
240
    $self->sort_fields(\%{$sort_fields{$self->index}});
240
241
    return $all_mappings{$self->index};
241
    return $all_mappings{$self->index};
242
}
242
}
243
243
Lines 527-534 sub marc_records_to_documents { Link Here
527
527
528
    my @record_documents;
528
    my @record_documents;
529
529
530
    my %auth_match_headings;
531
    if( $self->index eq 'authorities' ){
532
        my @auth_types = Koha::Authority::Types->search();
533
        %auth_match_headings = map { $_->authtypecode => $_->auth_tag_to_report } @auth_types;
534
    }
535
530
    foreach my $record (@{$records}) {
536
    foreach my $record (@{$records}) {
531
        my $record_document = {};
537
        my $record_document = {};
538
539
        if ( defined $record->{authtypecode} ){
540
            my $field = $record->{record}->field( $auth_match_headings{ $record->{authtypecode} } );
541
            my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
542
            push @{$record_document->{'match-heading'}}, $heading->search_form if $heading;
543
        }
544
        $record = $record->{record};
545
532
        my $mappings = $rules->{leader};
546
        my $mappings = $rules->{leader};
533
        if ($mappings) {
547
        if ($mappings) {
534
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
548
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
Lines 580-592 sub marc_records_to_documents { Link Here
580
                                }
594
                                }
581
                            );
595
                            );
582
                        }
596
                        }
583
                        if ( @{$mappings} && grep { $_->[0] eq 'match-heading'} @{$mappings} ){
584
                            # Used by the authority linker the match-heading field requires a specific syntax
585
                            # that is specified in C4/Heading
586
                            my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
587
                            next unless $heading;
588
                            push @{$record_document->{'match-heading'}}, $heading->search_form;
589
                        }
590
                    }
597
                    }
591
598
592
                    my $subfields_join_mappings = $data_field_rules->{subfields_join};
599
                    my $subfields_join_mappings = $data_field_rules->{subfields_join};
Lines 609-621 sub marc_records_to_documents { Link Here
609
                                    }
616
                                    }
610
                                );
617
                                );
611
                            }
618
                            }
612
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
613
                                # Used by the authority linker the match-heading field requires a specific syntax
614
                                # that is specified in C4/Heading
615
                                my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
616
                                next unless $heading;
617
                                push @{$record_document->{'match-heading'}}, $heading->search_form;
618
                            }
619
                        }
619
                        }
620
                    }
620
                    }
621
                }
621
                }
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +1 lines)
Lines 476-482 sub build_authorities_query_compat { Link Here
476
476
477
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
477
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
478
        push @indexes, $m;
478
        push @indexes, $m;
479
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '');
479
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
480
    }
480
    }
481
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
481
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
482
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
482
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
(-)a/admin/searchengine/elasticsearch/mappings.yaml (-24 lines)
Lines 695-724 authorities: Link Here
695
        sort: ~
695
        sort: ~
696
        suggestible: ''
696
        suggestible: ''
697
    type: ''
697
    type: ''
698
  Match-heading:
699
    label: Match-heading
700
    mappings:
701
      - facet: ''
702
        marc_field: 100(abcdefghjklmnopqrstvxyz)
703
        marc_type: marc21
704
        sort: ~
705
        suggestible: ''
706
      - facet: ''
707
        marc_field: 111(acdefghjklnpqstvxyz)
708
        marc_type: marc21
709
        sort: ~
710
        suggestible: ''
711
      - facet: ''
712
        marc_field: 100(abcdefghjklmnopqrstvxyz)
713
        marc_type: normarc
714
        sort: ~
715
        suggestible: ''
716
      - facet: ''
717
        marc_field: 111(acdefghjklnpqstvxyz)
718
        marc_type: normarc
719
        sort: ~
720
        suggestible: ''
721
    type: ''
722
  Match-heading-see-from:
698
  Match-heading-see-from:
723
    label: Match-heading-see-from
699
    label: Match-heading-see-from
724
    mappings:
700
    mappings:
(-)a/misc/search_tools/rebuild_elasticsearch.pl (-3 / +5 lines)
Lines 275-282 sub _do_reindex { Link Here
275
    my $commit_count = $commit;
275
    my $commit_count = $commit;
276
    my ( @id_buffer, @commit_buffer );
276
    my ( @id_buffer, @commit_buffer );
277
    while ( my $record = $next->() ) {
277
    while ( my $record = $next->() ) {
278
        my $id     = $record->id // $record->authid;
278
        my $id     = $record->id;
279
        my $record = $record->record;
279
        my $record_prepped;
280
        $record_prepped->{authtypecode} = $record->authtypecode if ref $record eq 'Koha::MetadataRecord::Authority';
281
        $record_prepped->{record} = $record->record;
280
        $count++;
282
        $count++;
281
        if ( $verbose == 1 ) {
283
        if ( $verbose == 1 ) {
282
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
284
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
Lines 285-291 sub _do_reindex { Link Here
285
        }
287
        }
286
288
287
        push @id_buffer,     $id;
289
        push @id_buffer,     $id;
288
        push @commit_buffer, $record;
290
        push @commit_buffer, $record_prepped;
289
        if ( !( --$commit_count ) ) {
291
        if ( !( --$commit_count ) ) {
290
            _log( 1, "Committing $commit records...\n" );
292
            _log( 1, "Committing $commit records...\n" );
291
            my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
293
            my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-56 / +26 lines)
Lines 21-26 use Test::More tests => 6; Link Here
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
24
use t::lib::TestBuilder;
24
25
25
use Test::MockModule;
26
use Test::MockModule;
26
27
Lines 31-36 use List::Util qw( any ); Link Here
31
use Koha::SearchEngine::Elasticsearch;
32
use Koha::SearchEngine::Elasticsearch;
32
use Koha::SearchEngine::Elasticsearch::Search;
33
use Koha::SearchEngine::Elasticsearch::Search;
33
34
35
my $schema = Koha::Database->new->schema;
36
$schema->storage->txn_begin;
37
34
subtest '_read_configuration() tests' => sub {
38
subtest '_read_configuration() tests' => sub {
35
39
36
    plan tests => 10;
40
    plan tests => 10;
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336
        MARC::Field->new('999', '', '', c => '1234568'),
340
        MARC::Field->new('999', '', '', c => '1234568'),
337
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
341
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
338
    );
342
    );
339
    my $records = [$marc_record_1, $marc_record_2];
343
    my $records = [
344
        { record => $marc_record_1 },
345
        { record => $marc_record_2 },
346
    ];
340
347
341
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
348
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
342
349
Lines 503-509 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
503
        $large_marc_record->append_fields($item_field);
510
        $large_marc_record->append_fields($item_field);
504
    }
511
    }
505
512
506
    $docs = $see->marc_records_to_documents([$large_marc_record]);
513
    $docs = $see->marc_records_to_documents([{ record => $large_marc_record }]);
507
514
508
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
515
    is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
509
516
Lines 616-622 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
616
        MARC::Field->new('999', '', '', c => '1234568'),
623
        MARC::Field->new('999', '', '', c => '1234568'),
617
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
624
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
618
    );
625
    );
619
    my $records = [$marc_record_1, $marc_record_2];
626
    my $records = [
627
        { record => $marc_record_1 },
628
        { record => $marc_record_2 },
629
    ];
620
630
621
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
631
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
622
632
Lines 642-647 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
642
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
652
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
643
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
653
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
644
654
655
    my $builder = t::lib::TestBuilder->new;
656
    my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
657
            auth_tag_to_report => '150'
658
        }
659
    });
660
645
    my @mappings = (
661
    my @mappings = (
646
        {
662
        {
647
            name => 'match',
663
            name => 'match',
Lines 652-708 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
652
            sort => 0,
668
            sort => 0,
653
            marc_type => 'marc21',
669
            marc_type => 'marc21',
654
            marc_field => '150(ae)',
670
            marc_field => '150(ae)',
655
        },
671
        }
656
        {
657
            name => 'heading',
658
            type => 'string',
659
            facet => 0,
660
            suggestible => 0,
661
            searchable => 1,
662
            sort => 0,
663
            marc_type => 'marc21',
664
            marc_field => '150a',
665
        },
666
        {
667
            name => 'heading',
668
            type => 'string',
669
            facet => 0,
670
            suggestible => 0,
671
            searchable => 1,
672
            sort => 0,
673
            marc_type => 'marc21',
674
            marc_field => '150(ae)',
675
        },
676
        {
677
            name => 'heading-main',
678
            type => 'string',
679
            facet => 0,
680
            suggestible => 0,
681
            searchable => 1,
682
            sort => 0,
683
            marc_type => 'marc21',
684
            marc_field => '150a',
685
        },
686
        {
687
            name => 'heading',
688
            type => 'string',
689
            facet => 0,
690
            suggestible => 0,
691
            searchable => 1,
692
            sort => 0,
693
            marc_type => 'marc21',
694
            marc_field => '150',
695
        },
696
        {
697
            name => 'match-heading',
698
            type => 'string',
699
            facet => 0,
700
            suggestible => 0,
701
            searchable => 1,
702
            sort => 0,
703
            marc_type => 'marc21',
704
            marc_field => '150',
705
        },
706
    );
672
    );
707
673
708
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
674
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 735-741 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
735
    $marc_record_2->append_fields(
701
    $marc_record_2->append_fields(
736
        MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
702
        MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
737
    );
703
    );
738
    my $records = [$marc_record_1, $marc_record_2];
704
    my $records = [
705
        { record => $marc_record_1, authtypecode=> $auth_type->authtypecode },
706
        { record => $marc_record_2, authtypecode=> $auth_type->authtypecode },
707
    ];
739
708
740
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
709
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
741
710
Lines 752-754 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
752
        "Second record match-heading should contain the correctly formatted heading without wrong subfield"
721
        "Second record match-heading should contain the correctly formatted heading without wrong subfield"
753
    );
722
    );
754
};
723
};
755
- 
724
725
$schema->storage->txn_rollback;

Return to bug 25273