Attachment #108382 for bug #24807

View | Details | Raw Unified | Return to bug 24807
Collapse All | Expand All

Lines 38-44 use Search::Elasticsearch; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-2 / +16 lines)
38	use Try::Tiny;	38	use Try::Tiny;
39	use YAML::Syck;	39	use YAML::Syck;
40		40
41	use List::Util qw( sum0 reduce );	41	use List::Util qw( sum0 reduce all );
42	use MARC::File::XML;	42	use MARC::File::XML;
43	use MIME::Base64;	43	use MIME::Base64;
44	use Encode qw(encode);	44	use Encode qw(encode);
Lines 213-218 sub get_elasticsearch_mappings { Link Here
213	$es_type = 'integer';	213	$es_type = 'integer';
214	} elsif ($type eq 'isbn' \|\| $type eq 'stdno') {	214	} elsif ($type eq 'isbn' \|\| $type eq 'stdno') {
215	$es_type = 'stdno';	215	$es_type = 'stdno';
		216	} elsif ($type eq 'year') {
		217	$es_type = 'year';
216	}	218	}
217		219
218	if ($search) {	220	if ($search) {
Lines 474-480 sub _process_mappings { Link Here
474	# with differing options for (possibly) mutating data	476	# with differing options for (possibly) mutating data
475	# so need a different copy for each	477	# so need a different copy for each
476	my $_data = $data;	478	my $_data = $data;
477	$record_document->{$target} //= [];
478	if (defined $options->{substr}) {	479	if (defined $options->{substr}) {
479	my ($start, $length) = @{$options->{substr}};	480	my ($start, $length) = @{$options->{substr}};
480	$_data = length($data) > $start ? substr $data, $start, $length : '';	481	$_data = length($data) > $start ? substr $data, $start, $length : '';
Lines 482-487 sub _process_mappings { Link Here
482	if (defined $options->{value_callbacks}) {	483	if (defined $options->{value_callbacks}) {
483	$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});	484	$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
484	}	485	}
		486	if (defined $options->{filter_callbacks}) {
		487	# Skip mapping unless all filter callbacks return true
		488	next unless all { $_->($_data) } @{$options->{filter_callbacks}};
		489	}
485	if (defined $options->{property}) {	490	if (defined $options->{property}) {
486	$_data = {	491	$_data = {
487	$options->{property} => $_data	492	$options->{property} => $_data
Lines 494-499 sub _process_mappings { Link Here
494	$_data = substr $_data, $nonfiling_chars;	499	$_data = substr $_data, $nonfiling_chars;
495	}	500	}
496	}	501	}
		502
		503	$record_document->{$target} //= [];
497	push @{$record_document->{$target}}, $_data;	504	push @{$record_document->{$target}}, $_data;
498	}	505	}
499	}	506	}
Lines 885-890 sub _field_mappings { Link Here
885	return $value ? 'true' : 'false';	892	return $value ? 'true' : 'false';
886	};	893	};
887	}	894	}
		895	elsif ($target_type eq 'year') {
		896	$default_options->{filter_callbacks} //= [];
		897	push @{$default_options->{filter_callbacks}}, sub {
		898	my ($value) = @_;
		899	return $value =~ /^\d+$/;
		900	};
		901	}
888		902
889	if ($search) {	903	if ($search) {
890	my $mapping = [$target_name, $default_options];	904	my $mapping = [$target_name, $default_options];

Lines 117-122 sub update_index { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (+3 lines)
117	type => 'data', # is just hard coded in Indexer.pm?	117	type => 'data', # is just hard coded in Indexer.pm?
118	body => \@body	118	body => \@body
119	);	119	);
		120	if ($response->{errors}) {
		121	carp "One or more ElasticSearch errors occured when indexing documents";
		122	}
120	}	123	}
121	return $response;	124	return $response;
122	}	125	}

Lines 25-30 search: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
25	type: integer	25	type: integer
26	null_value: 0	26	null_value: 0
27	ignore_malformed: true	27	ignore_malformed: true
		28	year:
		29	type: short
28	stdno:	30	stdno:
29	type: text	31	type: text
30	analyzer: analyzer_stdno	32	analyzer: analyzer_stdno

Lines 196-201 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
196	[% ELSE %]	196	[% ELSE %]
197	<option value="date">Date</option>	197	<option value="date">Date</option>
198	[% END %]	198	[% END %]
		199	[% IF search_field.type == "year" %]
		200	<option value="year" selected="selected">Year</option>
		201	[% ELSE %]
		202	<option value="year">Year</option>
		203	[% END %]
199	[% IF search_field.type == "number" %]	204	[% IF search_field.type == "number" %]
200	<option value="number" selected="selected">Number</option>	205	<option value="number" selected="selected">Number</option>
201	[% ELSE %]	206	[% ELSE %]

Lines 118-124 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +21 lines)
118		118
119	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	119	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
120		120
121	plan tests => 53;	121	plan tests => 56;
122		122
123	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	123	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
124	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');	124	t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 284-289 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
284	marc_type => 'marc21',	284	marc_type => 'marc21',
285	marc_field => '952l',	285	marc_field => '952l',
286	},	286	},
		287	{
		288	name => 'date-of-publication',
		289	type => 'year',
		290	facet => 0,
		291	suggestible => 0,
		292	searchable => 1,
		293	sort => 1,
		294	marc_type => 'marc21',
		295	marc_field => '008_/7-10',
		296	},
		297
287	);	298	);
288		299
289	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');	300	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 315-320 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
315	$marc_record_1->append_fields(	326	$marc_record_1->append_fields(
316	MARC::Field->new('001', '123'),	327	MARC::Field->new('001', '123'),
317	MARC::Field->new('007', 'ku'),	328	MARC::Field->new('007', 'ku'),
		329	MARC::Field->new('008', '901111s1962 xxk\|\|\|\| \|00\| \|\|eng c'),
318	MARC::Field->new('020', '', '', a => '1-56619-909-3'),	330	MARC::Field->new('020', '', '', a => '1-56619-909-3'),
319	MARC::Field->new('100', '', '', a => 'Author 1'),	331	MARC::Field->new('100', '', '', a => 'Author 1'),
320	MARC::Field->new('110', '', '', a => 'Corp Author'),	332	MARC::Field->new('110', '', '', a => 'Corp Author'),
Lines 330-335 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
330	my $marc_record_2 = MARC::Record->new();	342	my $marc_record_2 = MARC::Record->new();
331	$marc_record_2->leader(' cam 22 a 4500');	343	$marc_record_2->leader(' cam 22 a 4500');
332	$marc_record_2->append_fields(	344	$marc_record_2->append_fields(
		345	MARC::Field->new('008', '901111s19uu xxk\|\|\|\| \|00\| \|\|eng c'),
333	MARC::Field->new('100', '', '', a => 'Author 2'),	346	MARC::Field->new('100', '', '', a => 'Author 2'),
334	# MARC::Field->new('210', '', '', a => 'Title 2'),	347	# MARC::Field->new('210', '', '', a => 'Title 2'),
335	# MARC::Field->new('245', '', '', a => 'Title: second record'),	348	# MARC::Field->new('245', '', '', a => 'Title: second record'),
Lines 456-461 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
456	'First document uniform_title__sort field should contain the title with the first four initial characters removed'	469	'First document uniform_title__sort field should contain the title with the first four initial characters removed'
457	);	470	);
458		471
		472	# Tests for 'year' type and 'filter_callbacks'
		473	is(scalar @{$docs->[0]->{'date-of-publication'}}, 1, 'First document date-of-publication field should contain one value');
		474	is_deeply($docs->[0]->{'date-of-publication'}, ['1962'], 'First document date-of-publication field should be set correctly');
		475
459	# Second record:	476	# Second record:
460		477
461	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');	478	is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
Lines 480-485 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
480	'Second document local_classification__sort field should be set correctly'	497	'Second document local_classification__sort field should be set correctly'
481	);	498	);
482		499
		500	# Tests for 'year' type and 'filter_callbacks'
		501	ok(!(defined $docs->[1]->{'date-of-publication'}), "Second document invalid date-of-publication value should have been removed");
		502
483	# Mappings marc_type:	503	# Mappings marc_type:
484		504
485	ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");	505	ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
486	-

Return to bug 24807