Attachment #109261 for bug #25273

View | Details | Raw Unified | Return to bug 25273
Collapse All | Expand All

Lines 628-634 sub AddAuthority { Link Here

(-)a/C4/AuthoritiesMarc.pm (-1 / +1 lines)
628	$record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );	628	$record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
629	# Update	629	# Update
630	$dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;	630	$dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
631	ModZebra( $authid, 'specialUpdate', 'authorityserver', $record );	631	ModZebra( $authid, 'specialUpdate', 'authorityserver', { record => $record, authtypecode => $authtypecode } );
632		632
633	return ( $authid );	633	return ( $authid );
634	}	634	}





$server is the server that we want to update

$record is a hash including the authtypecode and updated MARC record. This is only used for
Elasticsearch authorities. If it's not supplied a biblio will be loaded from the database.

=cut


            }
        );
        if ( $op eq 'specialUpdate' ) {
            if ($record) {
                $indexer->update_index_background( [$biblionumber], [$record] );
            } else {
                $record = GetMarcBiblio({
                    biblionumber => $biblionumber,
                    embed_items  => 1 });
                $indexer->update_index_background( [$biblionumber], [{ record => $record }] );
            }
            my $records = [$record];
            $indexer->update_index_background( [$biblionumber], [$record] );
        }
        elsif ( $op eq 'recordDelete' ) {
            $indexer->delete_index_background( [$biblionumber] );

Lines 234-243 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-15 / +15 lines)
234	}	234	}
235	}	235	}
236	);	236	);
		237	$mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
237	$all_mappings{$self->index} = $mappings;	238	$all_mappings{$self->index} = $mappings;
238	}	239	}
239	$self->sort_fields(\%{$sort_fields{$self->index}});	240	$self->sort_fields(\%{$sort_fields{$self->index}});
240
241	return $all_mappings{$self->index};	241	return $all_mappings{$self->index};
242	}	242	}
243		243
Lines 527-534 sub marc_records_to_documents { Link Here
527		527
528	my @record_documents;	528	my @record_documents;
529		529
		530	my %auth_match_headings;
		531	if( $self->index eq 'authorities' ){
		532	my @auth_types = Koha::Authority::Types->search();
		533	%auth_match_headings = map { $_->authtypecode => $_->auth_tag_to_report } @auth_types;
		534	}
		535
530	foreach my $record (@{$records}) {	536	foreach my $record (@{$records}) {
531	my $record_document = {};	537	my $record_document = {};
		538
		539	if ( defined $record->{authtypecode} ){
		540	my $field = $record->{record}->field( $auth_match_headings{ $record->{authtypecode} } );
		541	my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
		542	push @{$record_document->{'match-heading'}}, $heading->search_form if $heading;
		543	}
		544	$record = $record->{record};
		545
532	my $mappings = $rules->{leader};	546	my $mappings = $rules->{leader};
533	if ($mappings) {	547	if ($mappings) {
534	$self->_process_mappings($mappings, $record->leader(), $record_document, {	548	$self->_process_mappings($mappings, $record->leader(), $record_document, {
Lines 580-592 sub marc_records_to_documents { Link Here
580	}	594	}
581	);	595	);
582	}	596	}
583	if ( @{$mappings} && grep { $_->[0] eq 'match-heading'} @{$mappings} ){
584	# Used by the authority linker the match-heading field requires a specific syntax
585	# that is specified in C4/Heading
586	my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
587	next unless $heading;
588	push @{$record_document->{'match-heading'}}, $heading->search_form;
589	}
590	}	597	}
591		598
592	my $subfields_join_mappings = $data_field_rules->{subfields_join};	599	my $subfields_join_mappings = $data_field_rules->{subfields_join};
Lines 609-621 sub marc_records_to_documents { Link Here
609	}	616	}
610	);	617	);
611	}	618	}
612	if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
613	# Used by the authority linker the match-heading field requires a specific syntax
614	# that is specified in C4/Heading
615	my $heading = C4::Heading->new_from_field( $field, undef, 1 ); #new auth heading
616	next unless $heading;
617	push @{$record_document->{'match-heading'}}, $heading->search_form;
618	}
619	}	619	}
620	}	620	}
621	}	621	}

Lines 476-482 sub build_authorities_query_compat { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +1 lines)
476		476
477	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;	477	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
478	push @indexes, $m;	478	push @indexes, $m;
479	warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} \|\| $m eq '');	479	warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} \|\| $m eq '' \|\| $m eq 'match-heading');
480	}	480	}
481	for ( my $i = 0 ; $i < @$value ; $i++ ) {	481	for ( my $i = 0 ; $i < @$value ; $i++ ) {
482	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches	482	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches




        sort: ~
        suggestible: ''
    type: ''
  Match-heading:
    label: Match-heading
    mappings:
      - facet: ''
        marc_field: 100(abcdefghjklmnopqrstvxyz)
        marc_type: marc21
        sort: ~
        suggestible: ''
      - facet: ''
        marc_field: 111(acdefghjklnpqstvxyz)
        marc_type: marc21
        sort: ~
        suggestible: ''
      - facet: ''
        marc_field: 100(abcdefghjklmnopqrstvxyz)
        marc_type: normarc
        sort: ~
        suggestible: ''
      - facet: ''
        marc_field: 111(acdefghjklnpqstvxyz)
        marc_type: normarc
        sort: ~
        suggestible: ''
    type: ''
  Match-heading-see-from:
    label: Match-heading-see-from
    mappings:

Lines 275-282 sub _do_reindex { Link Here

(-)a/misc/search_tools/rebuild_elasticsearch.pl (-3 / +5 lines)
275	my $commit_count = $commit;	275	my $commit_count = $commit;
276	my ( @id_buffer, @commit_buffer );	276	my ( @id_buffer, @commit_buffer );
277	while ( my $record = $next->() ) {	277	while ( my $record = $next->() ) {
278	my $id = $record->id // $record->authid;	278	my $id = $record->id;
279	my $record = $record->record;	279	my $record_prepped;
		280	$record_prepped->{authtypecode} = $record->authtypecode if ref $record eq 'Koha::MetadataRecord::Authority';
		281	$record_prepped->{record} = $record->record;
280	$count++;	282	$count++;
281	if ( $verbose == 1 ) {	283	if ( $verbose == 1 ) {
282	_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);	284	_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
Lines 285-291 sub _do_reindex { Link Here
285	}	287	}
286		288
287	push @id_buffer, $id;	289	push @id_buffer, $id;
288	push @commit_buffer, $record;	290	push @commit_buffer, $record_prepped;
289	if ( !( --$commit_count ) ) {	291	if ( !( --$commit_count ) ) {
290	_log( 1, "Committing $commit records...\n" );	292	_log( 1, "Committing $commit records...\n" );
291	my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );	293	my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );

Lines 21-26 use Test::More tests => 6; Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-56 / +26 lines)
21	use Test::Exception;	21	use Test::Exception;
22		22
23	use t::lib::Mocks;	23	use t::lib::Mocks;
		24	use t::lib::TestBuilder;
24		25
25	use Test::MockModule;	26	use Test::MockModule;
26		27
Lines 31-36 use List::Util qw( any ); Link Here
31	use Koha::SearchEngine::Elasticsearch;	32	use Koha::SearchEngine::Elasticsearch;
32	use Koha::SearchEngine::Elasticsearch::Search;	33	use Koha::SearchEngine::Elasticsearch::Search;
33		34
		35	my $schema = Koha::Database->new->schema;
		36	$schema->storage->txn_begin;
		37
34	subtest '_read_configuration() tests' => sub {	38	subtest '_read_configuration() tests' => sub {
35		39
36	plan tests => 10;	40	plan tests => 10;
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336	MARC::Field->new('999', '', '', c => '1234568'),	340	MARC::Field->new('999', '', '', c => '1234568'),
337	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),	341	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
338	);	342	);
339	my $records = [$marc_record_1, $marc_record_2];	343	my $records = [
		344	{ record => $marc_record_1 },
		345	{ record => $marc_record_2 },
		346	];
340		347
341	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first	348	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
342		349
Lines 503-509 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
503	$large_marc_record->append_fields($item_field);	510	$large_marc_record->append_fields($item_field);
504	}	511	}
505		512
506	$docs = $see->marc_records_to_documents([$large_marc_record]);	513	$docs = $see->marc_records_to_documents([{ record => $large_marc_record }]);
507		514
508	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');	515	is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
509		516
Lines 616-622 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
616	MARC::Field->new('999', '', '', c => '1234568'),	623	MARC::Field->new('999', '', '', c => '1234568'),
617	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),	624	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
618	);	625	);
619	my $records = [$marc_record_1, $marc_record_2];	626	my $records = [
		627	{ record => $marc_record_1 },
		628	{ record => $marc_record_2 },
		629	];
620		630
621	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first	631	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
622		632
Lines 642-647 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
642	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	652	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
643	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	653	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
644		654
		655	my $builder = t::lib::TestBuilder->new;
		656	my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
		657	auth_tag_to_report => '150'
		658	}
		659	});
		660
645	my @mappings = (	661	my @mappings = (
646	{	662	{
647	name => 'match',	663	name => 'match',
Lines 652-708 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
652	sort => 0,	668	sort => 0,
653	marc_type => 'marc21',	669	marc_type => 'marc21',
654	marc_field => '150(ae)',	670	marc_field => '150(ae)',
655	},	671	}
656	{
657	name => 'heading',
658	type => 'string',
659	facet => 0,
660	suggestible => 0,
661	searchable => 1,
662	sort => 0,
663	marc_type => 'marc21',
664	marc_field => '150a',
665	},
666	{
667	name => 'heading',
668	type => 'string',
669	facet => 0,
670	suggestible => 0,
671	searchable => 1,
672	sort => 0,
673	marc_type => 'marc21',
674	marc_field => '150(ae)',
675	},
676	{
677	name => 'heading-main',
678	type => 'string',
679	facet => 0,
680	suggestible => 0,
681	searchable => 1,
682	sort => 0,
683	marc_type => 'marc21',
684	marc_field => '150a',
685	},
686	{
687	name => 'heading',
688	type => 'string',
689	facet => 0,
690	suggestible => 0,
691	searchable => 1,
692	sort => 0,
693	marc_type => 'marc21',
694	marc_field => '150',
695	},
696	{
697	name => 'match-heading',
698	type => 'string',
699	facet => 0,
700	suggestible => 0,
701	searchable => 1,
702	sort => 0,
703	marc_type => 'marc21',
704	marc_field => '150',
705	},
706	);	672	);
707		673
708	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');	674	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 735-741 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
735	$marc_record_2->append_fields(	701	$marc_record_2->append_fields(
736	MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),	702	MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
737	);	703	);
738	my $records = [$marc_record_1, $marc_record_2];	704	my $records = [
		705	{ record => $marc_record_1, authtypecode=> $auth_type->authtypecode },
		706	{ record => $marc_record_2, authtypecode=> $auth_type->authtypecode },
		707	];
739		708
740	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first	709	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
741		710
Lines 752-754 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
752	"Second record match-heading should contain the correctly formatted heading without wrong subfield"	721	"Second record match-heading should contain the correctly formatted heading without wrong subfield"
753	);	722	);
754	};	723	};
755	-	724
		725	$schema->storage->txn_rollback;

Return to bug 25273

Lines 2497-2504 $op is specialUpdate or recordDelete, and is used to know what we want to do Link Here

(-)a/C4/Biblio.pm (-5 / +6 lines)
2497		2497
2498	$server is the server that we want to update	2498	$server is the server that we want to update
2499		2499
2500	$record is the update MARC record if it's available. If it's not supplied	2500	$record is a hash including the authtypecode and updated MARC record. This is only used for
2501	and is needed, it'll be loaded from the database.	2501	Elasticsearch authorities. If it's not supplied a biblio will be loaded from the database.
2502		2502
2503	=cut	2503	=cut
2504		2504
Lines 2518-2530 sub ModZebra { Link Here
2518	}	2518	}
2519	);	2519	);
2520	if ( $op eq 'specialUpdate' ) {	2520	if ( $op eq 'specialUpdate' ) {
2521	unless ($record) {	2521	if ($record) {
		2522	$indexer->update_index_background( [$biblionumber], [$record] );
		2523	} else {
2522	$record = GetMarcBiblio({	2524	$record = GetMarcBiblio({
2523	biblionumber => $biblionumber,	2525	biblionumber => $biblionumber,
2524	embed_items => 1 });	2526	embed_items => 1 });
		2527	$indexer->update_index_background( [$biblionumber], [{ record => $record }] );
2525	}	2528	}
2526	my $records = [$record];
2527	$indexer->update_index_background( [$biblionumber], [$record] );
2528	}	2529	}
2529	elsif ( $op eq 'recordDelete' ) {	2530	elsif ( $op eq 'recordDelete' ) {
2530	$indexer->delete_index_background( [$biblionumber] );	2531	$indexer->delete_index_background( [$biblionumber] );

Lines 695-724 authorities: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (-24 lines)
695	sort: ~	695	sort: ~
696	suggestible: ''	696	suggestible: ''
697	type: ''	697	type: ''
698	Match-heading:
699	label: Match-heading
700	mappings:
701	- facet: ''
702	marc_field: 100(abcdefghjklmnopqrstvxyz)
703	marc_type: marc21
704	sort: ~
705	suggestible: ''
706	- facet: ''
707	marc_field: 111(acdefghjklnpqstvxyz)
708	marc_type: marc21
709	sort: ~
710	suggestible: ''
711	- facet: ''
712	marc_field: 100(abcdefghjklmnopqrstvxyz)
713	marc_type: normarc
714	sort: ~
715	suggestible: ''
716	- facet: ''
717	marc_field: 111(acdefghjklnpqstvxyz)
718	marc_type: normarc
719	sort: ~
720	suggestible: ''
721	type: ''
722	Match-heading-see-from:	698	Match-heading-see-from:
723	label: Match-heading-see-from	699	label: Match-heading-see-from
724	mappings:	700	mappings: