Attachment #99691 for bug #22771

View | Details | Raw Unified | Return to bug 22771
Collapse All | Expand All




use MIME::Base64;
use Encode qw(encode);
use Business::ISBN;
use Scalar::Util qw(looks_like_number);

__PACKAGE__->mk_ro_accessors(qw( index ));
__PACKAGE__->mk_accessors(qw( sort_fields ));

    return $self->_sort_fields_accessor();
}

=head2 _process_mappings($mappings, $data, $record_document, $meta)

    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)


Hashref representing the Elasticsearch document on which mappings should be
applied.

=item C<$meta>

A hashref containing metadata useful for enforcing per mapping rules. For
example for providing extra context for mapping options, or treating mapping
targets differently depending on type (sort, search, facet etc). Combining
this metadata with the mapping options and metadata allows us to mutate the
data per mapping, or even replace it with other data retrieved from the
metadata context.

Current properties are:

C<altscript>: A boolean value indicating whether an alternate script presentation is being
processed.

C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
'subfield' or 'subfields_group'.

C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.

C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
is 'subfields_group'.

C<field>: The original marc field object.

=back

=cut

sub _process_mappings {
    my ($_self, $mappings, $data, $record_document, $meta) = @_;
    foreach my $mapping (@{$mappings}) {
        my ($target, $options) = @{$mapping};

        # Don't process sort fields for alternate scripts
        my $sort = $target =~ /__sort$/;
        if ($sort && $meta->{altscript}) {
            next;
        }


                $options->{property} => $_data
            }
        }
        if (defined $options->{nonfiling_characters_indicator}) {
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
            if ($nonfiling_chars) {
                $_data = substr $_data, $nonfiling_chars;
            }
        }
        push @{$record_document->{$target}}, $_data;
    }
}

        my $record_document = {};
        my $mappings = $rules->{leader};
        if ($mappings) {
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
                    altscript => 0,
                    data_source => 'leader'
                }
            );
        }
        foreach my $field ($record->fields()) {
            if ($field->is_control_field()) {
                my $mappings = $control_fields_rules->{$field->tag()};
                if ($mappings) {
                    $self->_process_mappings($mappings, $field->data(), $record_document, {
                            altscript => 0,
                            data_source => 'control_field',
                            field => $field
                        }
                    );
                }
            }
            else {

                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
                        }
                        if (@{$mappings}) {
                            $self->_process_mappings($mappings, $data, $record_document, {
                                    altscript => $altscript,
                                    data_source => 'subfield',
                                    code => $code,
                                    field => $field
                                }
                            );
                        }
                    }


                                )
                            );
                            if ($data) {
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
                                        altscript => $altscript,
                                        data_source => 'subfields_group',
                                        codes => $subfields_group,
                                        field => $field
                                    }
                                );
                            }
                        }
                    }

            push @{$mapping}, {%{$default_options}, property => 'input'};
        }
        else {
            # Important! Make shallow clone, or we end up with the same hashref
            # shared by all mappings
            push @{$mapping}, {%{$default_options}};
        }
        push @mappings, $mapping;
    }

                    push @{$rules->{data_fields}->{$field_tag}->{subfields}->{$subfield}}, @mappings;
                }
                foreach my $subfield_group (@subfield_groups) {
                   $rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group} //= [];
                    push @{$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group}}, @mappings;
                }
            }

            );
        }
    });

    # Marc-flavour specific rule tweaks, could/should also provide hook for this
    if ($marcflavour eq 'marc21') {
        # Nonfiling characters processing for sort fields
        my %title_fields;
        if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
            # Format is: nonfiling characters indicator => field names list
            %title_fields = (
                1 => [630, 730, 740],
                2 => [130, 222, 240, 242, 243, 440, 830]
            );
        }
        elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
            %title_fields = (
                1 => [730],
                2 => [130, 430, 530]
            );
        }
        foreach my $indicator (keys %title_fields) {
            foreach my $field_tag (@{$title_fields{$indicator}}) {
                my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
                foreach my $mapping (@{$mappings}) {
                    if ($mapping->[0] =~ /__sort$/) {
                        # Mark this as to be processed for nonfiling characters indicator
                        # later on in _process_mappings
                        $mapping->[1]->{nonfiling_characters_indicator} = $indicator;
                    }
                }
            }
        }
    }

    return $rules;
}


Lines 117-123 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +24 lines)
117		117
118	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	118	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119		119
120	plan tests => 51;	120	plan tests => 53;
121		121
122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	123	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 193-198 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
193	marc_type => 'marc21',	193	marc_type => 'marc21',
194	marc_field => '220',	194	marc_field => '220',
195	},	195	},
		196	{
		197	name => 'uniform_title',
		198	type => 'string',
		199	facet => 0,
		200	suggestible => 0,
		201	searchable => 1,
		202	sort => 1,
		203	marc_type => 'marc21',
		204	marc_field => '240a',
		205	},
196	{	206	{
197	name => 'title_wildcard',	207	name => 'title_wildcard',
198	type => 'string',	208	type => 'string',
Lines 308-313 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
308	MARC::Field->new('100', '', '', a => 'Author 1'),	318	MARC::Field->new('100', '', '', a => 'Author 1'),
309	MARC::Field->new('110', '', '', a => 'Corp Author'),	319	MARC::Field->new('110', '', '', a => 'Corp Author'),
310	MARC::Field->new('210', '', '', a => 'Title 1'),	320	MARC::Field->new('210', '', '', a => 'Title 1'),
		321	MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
311	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),	322	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
312	MARC::Field->new('999', '', '', c => '1234567'),	323	MARC::Field->new('999', '', '', c => '1234567'),
313	# ' ' for testing trimming of white space in boolean value callback:	324	# ' ' for testing trimming of white space in boolean value callback:
Lines 432-437 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
432	'First document local_classification field should be set correctly'	443	'First document local_classification field should be set correctly'
433	);	444	);
434		445
		446	# Nonfiling characters for sort fields
		447	is_deeply(
		448	$docs->[0]->{uniform_title},
		449	['The uniform title with nonfiling indicator'],
		450	'First document uniform_title field should contain the title verbatim'
		451	);
		452	is_deeply(
		453	$docs->[0]->{uniform_title__sort},
		454	['uniform title with nonfiling indicator'],
		455	'First document uniform_title__sort field should contain the title with the first four initial characters removed'
		456	);
		457
435	# Second record:	458	# Second record:
436		459
437	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');	460	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
438	-

Return to bug 22771

Lines 41-46 use MARC::File::XML; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-11 / +93 lines)
41	use MIME::Base64;	41	use MIME::Base64;
42	use Encode qw(encode);	42	use Encode qw(encode);
43	use Business::ISBN;	43	use Business::ISBN;
		44	use Scalar::Util qw(looks_like_number);
44		45
45	__PACKAGE__->mk_ro_accessors(qw( index ));	46	__PACKAGE__->mk_ro_accessors(qw( index ));
46	__PACKAGE__->mk_accessors(qw( sort_fields ));	47	__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 346-352 sub sort_fields { Link Here
346	return $self->_sort_fields_accessor();	347	return $self->_sort_fields_accessor();
347	}	348	}
348		349
349	=head2 _process_mappings($mappings, $data, $record_document, $altscript)	350	=head2 _process_mappings($mappings, $data, $record_document, $meta)
350		351
351	$self->_process_mappings($mappings, $marc_field_data, $record_document, 0)	352	$self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352		353
Lines 374-396 The source data from a MARC record field. Link Here
374	Hashref representing the Elasticsearch document on which mappings should be	375	Hashref representing the Elasticsearch document on which mappings should be
375	applied.	376	applied.
376		377
377	=item C<$altscript>	378	=item C<$meta>
378		379
379	A boolean value indicating whether an alternate script presentation is being	380	A hashref containing metadata useful for enforcing per mapping rules. For
		381	example for providing extra context for mapping options, or treating mapping
		382	targets differently depending on type (sort, search, facet etc). Combining
		383	this metadata with the mapping options and metadata allows us to mutate the
		384	data per mapping, or even replace it with other data retrieved from the
		385	metadata context.
		386
		387	Current properties are:
		388
		389	C<altscript>: A boolean value indicating whether an alternate script presentation is being
380	processed.	390	processed.
381		391
		392	C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
		393	'subfield' or 'subfields_group'.
		394
		395	C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.
		396
		397	C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
		398	is 'subfields_group'.
		399
		400	C<field>: The original marc field object.
		401
382	=back	402	=back
383		403
384	=cut	404	=cut
385		405
386	sub _process_mappings {	406	sub _process_mappings {
387	my ($_self, $mappings, $data, $record_document, $altscript) = @_;	407	my ($_self, $mappings, $data, $record_document, $meta) = @_;
388	foreach my $mapping (@{$mappings}) {	408	foreach my $mapping (@{$mappings}) {
389	my ($target, $options) = @{$mapping};	409	my ($target, $options) = @{$mapping};
390		410
391	# Don't process sort fields for alternate scripts	411	# Don't process sort fields for alternate scripts
392	my $sort = $target =~ /__sort$/;	412	my $sort = $target =~ /__sort$/;
393	if ($sort && $altscript) {	413	if ($sort && $meta->{altscript}) {
394	next;	414	next;
395	}	415	}
396		416
Lines 411-416 sub _process_mappings { Link Here
411	$options->{property} => $_data	431	$options->{property} => $_data
412	}	432	}
413	}	433	}
		434	if (defined $options->{nonfiling_characters_indicator}) {
		435	my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
		436	$nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
		437	if ($nonfiling_chars) {
		438	$_data = substr $_data, $nonfiling_chars;
		439	}
		440	}
414	push @{$record_document->{$target}}, $_data;	441	push @{$record_document->{$target}}, $_data;
415	}	442	}
416	}	443	}
Lines 448-460 sub marc_records_to_documents { Link Here
448	my $record_document = {};	475	my $record_document = {};
449	my $mappings = $rules->{leader};	476	my $mappings = $rules->{leader};
450	if ($mappings) {	477	if ($mappings) {
451	$self->_process_mappings($mappings, $record->leader(), $record_document, 0);	478	$self->_process_mappings($mappings, $record->leader(), $record_document, {
		479	altscript => 0,
		480	data_source => 'leader'
		481	}
		482	);
452	}	483	}
453	foreach my $field ($record->fields()) {	484	foreach my $field ($record->fields()) {
454	if ($field->is_control_field()) {	485	if ($field->is_control_field()) {
455	my $mappings = $control_fields_rules->{$field->tag()};	486	my $mappings = $control_fields_rules->{$field->tag()};
456	if ($mappings) {	487	if ($mappings) {
457	$self->_process_mappings($mappings, $field->data(), $record_document, 0);	488	$self->_process_mappings($mappings, $field->data(), $record_document, {
		489	altscript => 0,
		490	data_source => 'control_field',
		491	field => $field
		492	}
		493	);
458	}	494	}
459	}	495	}
460	else {	496	else {
Lines 481-487 sub marc_records_to_documents { Link Here
481	$mappings = [@{$mappings}, @{$wildcard_mappings}];	517	$mappings = [@{$mappings}, @{$wildcard_mappings}];
482	}	518	}
483	if (@{$mappings}) {	519	if (@{$mappings}) {
484	$self->_process_mappings($mappings, $data, $record_document, $altscript);	520	$self->_process_mappings($mappings, $data, $record_document, {
		521	altscript => $altscript,
		522	data_source => 'subfield',
		523	code => $code,
		524	field => $field
		525	}
		526	);
485	}	527	}
486	}	528	}
487		529
Lines 497-503 sub marc_records_to_documents { Link Here
497	)	539	)
498	);	540	);
499	if ($data) {	541	if ($data) {
500	$self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);	542	$self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
		543	altscript => $altscript,
		544	data_source => 'subfields_group',
		545	codes => $subfields_group,
		546	field => $field
		547	}
		548	);
501	}	549	}
502	}	550	}
503	}	551	}
Lines 786-792 sub _field_mappings { Link Here
786	push @{$mapping}, {%{$default_options}, property => 'input'};	834	push @{$mapping}, {%{$default_options}, property => 'input'};
787	}	835	}
788	else {	836	else {
789	push @{$mapping}, $default_options;	837	# Important! Make shallow clone, or we end up with the same hashref
		838	# shared by all mappings
		839	push @{$mapping}, {%{$default_options}};
790	}	840	}
791	push @mappings, $mapping;	841	push @mappings, $mapping;
792	}	842	}
Lines 905-911 sub _get_marc_mapping_rules { Link Here
905	push @{$rules->{data_fields}->{$field_tag}->{subfields}->{$subfield}}, @mappings;	955	push @{$rules->{data_fields}->{$field_tag}->{subfields}->{$subfield}}, @mappings;
906	}	956	}
907	foreach my $subfield_group (@subfield_groups) {	957	foreach my $subfield_group (@subfield_groups) {
908	$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group} //= [];	958	$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group} //= [];
909	push @{$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group}}, @mappings;	959	push @{$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group}}, @mappings;
910	}	960	}
911	}	961	}
Lines 921-926 sub _get_marc_mapping_rules { Link Here
921	);	971	);
922	}	972	}
923	});	973	});
		974
		975	# Marc-flavour specific rule tweaks, could/should also provide hook for this
		976	if ($marcflavour eq 'marc21') {
		977	# Nonfiling characters processing for sort fields
		978	my %title_fields;
		979	if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
		980	# Format is: nonfiling characters indicator => field names list
		981	%title_fields = (
		982	1 => [630, 730, 740],
		983	2 => [130, 222, 240, 242, 243, 440, 830]
		984	);
		985	}
		986	elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
		987	%title_fields = (
		988	1 => [730],
		989	2 => [130, 430, 530]
		990	);
		991	}
		992	foreach my $indicator (keys %title_fields) {
		993	foreach my $field_tag (@{$title_fields{$indicator}}) {
		994	my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
		995	foreach my $mapping (@{$mappings}) {
		996	if ($mapping->[0] =~ /__sort$/) {
		997	# Mark this as to be processed for nonfiling characters indicator
		998	# later on in _process_mappings
		999	$mapping->[1]->{nonfiling_characters_indicator} = $indicator;
		1000	}
		1001	}
		1002	}
		1003	}
		1004	}
		1005
924	return $rules;	1006	return $rules;
925	}	1007	}
926		1008