Attachment #101551 for bug #22771

View | Details | Raw Unified | Return to bug 22771
Collapse All | Expand All




use MIME::Base64;
use Encode qw(encode);
use Business::ISBN;
use Scalar::Util qw(looks_like_number);

__PACKAGE__->mk_ro_accessors(qw( index ));
__PACKAGE__->mk_accessors(qw( sort_fields ));

    return $self->_sort_fields_accessor();
}

=head2 _process_mappings($mappings, $data, $record_document, $meta)

    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)


Hashref representing the Elasticsearch document on which mappings should be
applied.

=item C<$meta>

A hashref containing metadata useful for enforcing per mapping rules. For
example for providing extra context for mapping options, or treating mapping
targets differently depending on type (sort, search, facet etc). Combining
this metadata with the mapping options and metadata allows us to mutate the
data per mapping, or even replace it with other data retrieved from the
metadata context.

Current properties are:

C<altscript>: A boolean value indicating whether an alternate script presentation is being
processed.

C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
'subfield' or 'subfields_group'.

C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.

C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
is 'subfields_group'.

C<field>: The original C<MARC::Record> object.

=back

=cut

sub _process_mappings {
    my ($_self, $mappings, $data, $record_document, $meta) = @_;
    foreach my $mapping (@{$mappings}) {
        my ($target, $options) = @{$mapping};

        # Don't process sort fields for alternate scripts
        my $sort = $target =~ /__sort$/;
        if ($sort && $meta->{altscript}) {
            next;
        }


                $options->{property} => $_data
            }
        }
        if (defined $options->{nonfiling_characters_indicator}) {
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
            if ($nonfiling_chars) {
                $_data = substr $_data, $nonfiling_chars;
            }
        }
        push @{$record_document->{$target}}, $_data;
    }
}

        my $record_document = {};
        my $mappings = $rules->{leader};
        if ($mappings) {
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
                    altscript => 0,
                    data_source => 'leader'
                }
            );
        }
        foreach my $field ($record->fields()) {
            if ($field->is_control_field()) {
                my $mappings = $control_fields_rules->{$field->tag()};
                if ($mappings) {
                    $self->_process_mappings($mappings, $field->data(), $record_document, {
                            altscript => 0,
                            data_source => 'control_field',
                            field => $field
                        }
                    );
                }
            }
            else {

                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
                        }
                        if (@{$mappings}) {
                            $self->_process_mappings($mappings, $data, $record_document, {
                                    altscript => $altscript,
                                    data_source => 'subfield',
                                    code => $code,
                                    field => $field
                                }
                            );
                        }
                        if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
                            # Used by the authority linker the match-heading field requires a specific syntax

                                )
                            );
                            if ($data) {
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
                                        altscript => $altscript,
                                        data_source => 'subfields_group',
                                        codes => $subfields_group,
                                        field => $field
                                    }
                                );
                            }
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
                                # Used by the authority linker the match-heading field requires a specific syntax

            push @{$mapping}, {%{$default_options}, property => 'input'};
        }
        else {
            # Important! Make shallow clone, or we end up with the same hashref
            # shared by all mappings
            push @{$mapping}, {%{$default_options}};
        }
        push @mappings, $mapping;
    }

            );
        }
    });

    # Marc-flavour specific rule tweaks, could/should also provide hook for this
    if ($marcflavour eq 'marc21') {
        # Nonfiling characters processing for sort fields
        my %title_fields;
        if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
            # Format is: nonfiling characters indicator => field names list
            %title_fields = (
                1 => [630, 730, 740],
                2 => [130, 222, 240, 242, 243, 440, 830]
            );
        }
        elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
            %title_fields = (
                1 => [730],
                2 => [130, 430, 530]
            );
        }
        foreach my $indicator (keys %title_fields) {
            foreach my $field_tag (@{$title_fields{$indicator}}) {
                my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
                foreach my $mapping (@{$mappings}) {
                    if ($mapping->[0] =~ /__sort$/) {
                        # Mark this as to be processed for nonfiling characters indicator
                        # later on in _process_mappings
                        $mapping->[1]->{nonfiling_characters_indicator} = $indicator;
                    }
                }
            }
        }
    }

    return $rules;
}


Lines 118-124 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +24 lines)
118		118
119	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	119	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
120		120
121	plan tests => 51;	121	plan tests => 53;
122		122
123	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	123	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
124	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	124	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 195-200 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
195	marc_field => '220',	195	marc_field => '220',
196	},	196	},
197	{	197	{
		198	name => 'uniform_title',
		199	type => 'string',
		200	facet => 0,
		201	suggestible => 0,
		202	searchable => 1,
		203	sort => 1,
		204	marc_type => 'marc21',
		205	marc_field => '240a',
		206	},
		207	{
198	name => 'title_wildcard',	208	name => 'title_wildcard',
199	type => 'string',	209	type => 'string',
200	facet => 0,	210	facet => 0,
Lines 309-314 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
309	MARC::Field->new('100', '', '', a => 'Author 1'),	319	MARC::Field->new('100', '', '', a => 'Author 1'),
310	MARC::Field->new('110', '', '', a => 'Corp Author'),	320	MARC::Field->new('110', '', '', a => 'Corp Author'),
311	MARC::Field->new('210', '', '', a => 'Title 1'),	321	MARC::Field->new('210', '', '', a => 'Title 1'),
		322	MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
312	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),	323	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
313	MARC::Field->new('999', '', '', c => '1234567'),	324	MARC::Field->new('999', '', '', c => '1234567'),
314	# ' ' for testing trimming of white space in boolean value callback:	325	# ' ' for testing trimming of white space in boolean value callback:
Lines 433-438 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
433	'First document local_classification field should be set correctly'	444	'First document local_classification field should be set correctly'
434	);	445	);
435		446
		447	# Nonfiling characters for sort fields
		448	is_deeply(
		449	$docs->[0]->{uniform_title},
		450	['The uniform title with nonfiling indicator'],
		451	'First document uniform_title field should contain the title verbatim'
		452	);
		453	is_deeply(
		454	$docs->[0]->{uniform_title__sort},
		455	['uniform title with nonfiling indicator'],
		456	'First document uniform_title__sort field should contain the title with the first four initial characters removed'
		457	);
		458
436	# Second record:	459	# Second record:
437		460
438	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');	461	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
439	-

Return to bug 22771

Lines 42-47 use MARC::File::XML; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-10 / +92 lines)
42	use MIME::Base64;	42	use MIME::Base64;
43	use Encode qw(encode);	43	use Encode qw(encode);
44	use Business::ISBN;	44	use Business::ISBN;
		45	use Scalar::Util qw(looks_like_number);
45		46
46	__PACKAGE__->mk_ro_accessors(qw( index ));	47	__PACKAGE__->mk_ro_accessors(qw( index ));
47	__PACKAGE__->mk_accessors(qw( sort_fields ));	48	__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 346-352 sub sort_fields { Link Here
346	return $self->_sort_fields_accessor();	347	return $self->_sort_fields_accessor();
347	}	348	}
348		349
349	=head2 _process_mappings($mappings, $data, $record_document, $altscript)	350	=head2 _process_mappings($mappings, $data, $record_document, $meta)
350		351
351	$self->_process_mappings($mappings, $marc_field_data, $record_document, 0)	352	$self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352		353
Lines 374-396 The source data from a MARC record field. Link Here
374	Hashref representing the Elasticsearch document on which mappings should be	375	Hashref representing the Elasticsearch document on which mappings should be
375	applied.	376	applied.
376		377
377	=item C<$altscript>	378	=item C<$meta>
378		379
379	A boolean value indicating whether an alternate script presentation is being	380	A hashref containing metadata useful for enforcing per mapping rules. For
		381	example for providing extra context for mapping options, or treating mapping
		382	targets differently depending on type (sort, search, facet etc). Combining
		383	this metadata with the mapping options and metadata allows us to mutate the
		384	data per mapping, or even replace it with other data retrieved from the
		385	metadata context.
		386
		387	Current properties are:
		388
		389	C<altscript>: A boolean value indicating whether an alternate script presentation is being
380	processed.	390	processed.
381		391
		392	C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
		393	'subfield' or 'subfields_group'.
		394
		395	C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.
		396
		397	C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
		398	is 'subfields_group'.
		399
		400	C<field>: The original C<MARC::Record> object.
		401
382	=back	402	=back
383		403
384	=cut	404	=cut
385		405
386	sub _process_mappings {	406	sub _process_mappings {
387	my ($_self, $mappings, $data, $record_document, $altscript) = @_;	407	my ($_self, $mappings, $data, $record_document, $meta) = @_;
388	foreach my $mapping (@{$mappings}) {	408	foreach my $mapping (@{$mappings}) {
389	my ($target, $options) = @{$mapping};	409	my ($target, $options) = @{$mapping};
390		410
391	# Don't process sort fields for alternate scripts	411	# Don't process sort fields for alternate scripts
392	my $sort = $target =~ /__sort$/;	412	my $sort = $target =~ /__sort$/;
393	if ($sort && $altscript) {	413	if ($sort && $meta->{altscript}) {
394	next;	414	next;
395	}	415	}
396		416
Lines 411-416 sub _process_mappings { Link Here
411	$options->{property} => $_data	431	$options->{property} => $_data
412	}	432	}
413	}	433	}
		434	if (defined $options->{nonfiling_characters_indicator}) {
		435	my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
		436	$nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
		437	if ($nonfiling_chars) {
		438	$_data = substr $_data, $nonfiling_chars;
		439	}
		440	}
414	push @{$record_document->{$target}}, $_data;	441	push @{$record_document->{$target}}, $_data;
415	}	442	}
416	}	443	}
Lines 448-460 sub marc_records_to_documents { Link Here
448	my $record_document = {};	475	my $record_document = {};
449	my $mappings = $rules->{leader};	476	my $mappings = $rules->{leader};
450	if ($mappings) {	477	if ($mappings) {
451	$self->_process_mappings($mappings, $record->leader(), $record_document, 0);	478	$self->_process_mappings($mappings, $record->leader(), $record_document, {
		479	altscript => 0,
		480	data_source => 'leader'
		481	}
		482	);
452	}	483	}
453	foreach my $field ($record->fields()) {	484	foreach my $field ($record->fields()) {
454	if ($field->is_control_field()) {	485	if ($field->is_control_field()) {
455	my $mappings = $control_fields_rules->{$field->tag()};	486	my $mappings = $control_fields_rules->{$field->tag()};
456	if ($mappings) {	487	if ($mappings) {
457	$self->_process_mappings($mappings, $field->data(), $record_document, 0);	488	$self->_process_mappings($mappings, $field->data(), $record_document, {
		489	altscript => 0,
		490	data_source => 'control_field',
		491	field => $field
		492	}
		493	);
458	}	494	}
459	}	495	}
460	else {	496	else {
Lines 480-486 sub marc_records_to_documents { Link Here
480	$mappings = [@{$mappings}, @{$wildcard_mappings}];	516	$mappings = [@{$mappings}, @{$wildcard_mappings}];
481	}	517	}
482	if (@{$mappings}) {	518	if (@{$mappings}) {
483	$self->_process_mappings($mappings, $data, $record_document, $altscript);	519	$self->_process_mappings($mappings, $data, $record_document, {
		520	altscript => $altscript,
		521	data_source => 'subfield',
		522	code => $code,
		523	field => $field
		524	}
		525	);
484	}	526	}
485	if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){	527	if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
486	# Used by the authority linker the match-heading field requires a specific syntax	528	# Used by the authority linker the match-heading field requires a specific syntax
Lines 503-509 sub marc_records_to_documents { Link Here
503	)	545	)
504	);	546	);
505	if ($data) {	547	if ($data) {
506	$self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);	548	$self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
		549	altscript => $altscript,
		550	data_source => 'subfields_group',
		551	codes => $subfields_group,
		552	field => $field
		553	}
		554	);
507	}	555	}
508	if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){	556	if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
509	# Used by the authority linker the match-heading field requires a specific syntax	557	# Used by the authority linker the match-heading field requires a specific syntax
Lines 799-805 sub _field_mappings { Link Here
799	push @{$mapping}, {%{$default_options}, property => 'input'};	847	push @{$mapping}, {%{$default_options}, property => 'input'};
800	}	848	}
801	else {	849	else {
802	push @{$mapping}, $default_options;	850	# Important! Make shallow clone, or we end up with the same hashref
		851	# shared by all mappings
		852	push @{$mapping}, {%{$default_options}};
803	}	853	}
804	push @mappings, $mapping;	854	push @mappings, $mapping;
805	}	855	}
Lines 934-939 sub _get_marc_mapping_rules { Link Here
934	);	984	);
935	}	985	}
936	});	986	});
		987
		988	# Marc-flavour specific rule tweaks, could/should also provide hook for this
		989	if ($marcflavour eq 'marc21') {
		990	# Nonfiling characters processing for sort fields
		991	my %title_fields;
		992	if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
		993	# Format is: nonfiling characters indicator => field names list
		994	%title_fields = (
		995	1 => [630, 730, 740],
		996	2 => [130, 222, 240, 242, 243, 440, 830]
		997	);
		998	}
		999	elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
		1000	%title_fields = (
		1001	1 => [730],
		1002	2 => [130, 430, 530]
		1003	);
		1004	}
		1005	foreach my $indicator (keys %title_fields) {
		1006	foreach my $field_tag (@{$title_fields{$indicator}}) {
		1007	my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
		1008	foreach my $mapping (@{$mappings}) {
		1009	if ($mapping->[0] =~ /__sort$/) {
		1010	# Mark this as to be processed for nonfiling characters indicator
		1011	# later on in _process_mappings
		1012	$mapping->[1]->{nonfiling_characters_indicator} = $indicator;
		1013	}
		1014	}
		1015	}
		1016	}
		1017	}
		1018
937	return $rules;	1019	return $rules;
938	}	1020	}
939		1021