Attachment #164689 for bug #31652

View | Details | Raw Unified | Return to bug 31652
Collapse All | Expand All

Lines 212-217 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (+22 lines)
212	$es_type = 'year';	212	$es_type = 'year';
213	} elsif ($type eq 'callnumber') {	213	} elsif ($type eq 'callnumber') {
214	$es_type = 'cn_sort';	214	$es_type = 'cn_sort';
		215	} elsif ($type eq 'geo_point') {
		216	$es_type = 'geo_point';
		217	}
		218
		219	if ($type eq 'geo_point') {
		220	$name =~ s/_(lat\|lon)$//;
215	}	221	}
216		222
217	if ($search) {	223	if ($search) {
Lines 735-740 sub marc_records_to_documents { Link Here
735	}	741	}
736	}	742	}
737		743
		744	foreach my $field (@{$rules->{geo_point}}) {
		745	next unless $record_document->{$field};
		746	my $geofield = $field;
		747	$geofield =~ s/_(lat\|lon)$//;
		748	my $axis = $1;
		749	my $vals = $record_document->{$field};
		750	for my $i (0 .. @$vals - 1) {
		751	my $val = $record_document->{$field}[$i];
		752	$record_document->{$geofield}[$i]{$axis} = $val;
		753	}
		754	delete $record_document->{$field};
		755	}
		756
738	# Remove duplicate values and collapse sort fields	757	# Remove duplicate values and collapse sort fields
739	foreach my $field (keys %{$record_document}) {	758	foreach my $field (keys %{$record_document}) {
740	if (ref($record_document->{$field}) eq 'ARRAY') {	759	if (ref($record_document->{$field}) eq 'ARRAY') {
Lines 1070-1075 sub _get_marc_mapping_rules { Link Here
1070	elsif ($type eq 'isbn') {	1089	elsif ($type eq 'isbn') {
1071	push @{$rules->{isbn}}, $name;	1090	push @{$rules->{isbn}}, $name;
1072	}	1091	}
		1092	elsif ($type eq 'geo_point') {
		1093	push @{$rules->{geo_point}}, $name;
		1094	}
1073	elsif ($type eq 'boolean') {	1095	elsif ($type eq 'boolean') {
1074	# boolean gets special handling, if value doesn't exist for a field,	1096	# boolean gets special handling, if value doesn't exist for a field,
1075	# it is set to false	1097	# it is set to false

Lines 137-142 our %index_field_convert = ( Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +50 lines)
137	);	137	);
138	my $field_name_pattern = '[\w\-]+';	138	my $field_name_pattern = '[\w\-]+';
139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";	139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";
		140	my $es_advanced_searches = [];
140		141
141	=head2 get_index_field_convert	142	=head2 get_index_field_convert
142		143
Lines 245-250 sub build_query { Link Here
245	or $display_library_facets eq 'holding' ) {	246	or $display_library_facets eq 'holding' ) {
246	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };	247	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };
247	}	248	}
		249
		250	$res = _rebuild_to_es_advanced_query($res) if @$es_advanced_searches ;
248	return $res;	251	return $res;
249	}	252	}
250		253
Lines 930-935 operand. Link Here
930		933
931	sub _create_query_string {	934	sub _create_query_string {
932	my ( $self, @queries ) = @_;	935	my ( $self, @queries ) = @_;
		936	$es_advanced_searches = [];
		937	my @string_queries;
		938	foreach my $q (@queries) {
		939	if ($q->{field} && $q->{field} eq 'geolocation') {
		940	push(@$es_advanced_searches, $q);
		941	} else {
		942	push(@string_queries, $q)
		943	}
		944	}
		945
		946	@queries = @string_queries;
933		947
934	map {	948	map {
935	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';	949	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';
Lines 1083-1089 sub _fix_limit_special_cases { Link Here
1083		1097
1084	my @new_lim;	1098	my @new_lim;
1085	foreach my $l (@$limits) {	1099	foreach my $l (@$limits) {
1086
1087	# This is set up by opac-search.pl	1100	# This is set up by opac-search.pl
1088	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {	1101	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {
1089	my ( $start, $end ) =	1102	my ( $start, $end ) =
Lines 1407-1410 sub _is_safe_to_auto_truncate { Link Here
1407	return 1;	1420	return 1;
1408	}	1421	}
1409		1422
		1423	sub _rebuild_to_es_advanced_query {
		1424	my ($res) = @_;
		1425	my $query_string = $res->{query}->{query_string};
		1426	$query_string->{query} = '*' unless $query_string->{query};
		1427	delete $res->{query}->{query_string};
		1428
		1429	my %filter;
		1430	for my $advanced_query (@$es_advanced_searches) {
		1431	if ( $advanced_query->{field} eq 'geolocation') {
		1432	my ($lat, $lon, $distance) = map { $_ =~ /:(.)\/ } split('\s+', $advanced_query->{operand});
		1433	$filter{geo_distance} = {
		1434	distance => $distance,
		1435	geolocation => {
		1436	lat => $lat,
		1437	lon => $lon,
		1438	}
		1439	};
		1440	}
		1441	else {
		1442	warn "unknown advanced ElasticSearch query: ".join(', ',%$advanced_query);
		1443	}
		1444	}
		1445
		1446	$res->{query} = {
		1447	bool => {
		1448	must => {
		1449	query_string => $query_string
		1450	},
		1451	filter => \%filter,
		1452	}
		1453	};
		1454
		1455	return $res;
		1456	}
		1457
		1458
1410	1;	1459	1;

Lines 91-96 sub search { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+5 lines)
91	$query->{from} = $page * $query->{size};	91	$query->{from} = $page * $query->{size};
92	}	92	}
93	my $elasticsearch = $self->get_elasticsearch();	93	my $elasticsearch = $self->get_elasticsearch();
		94
		95	# FixMe: investigate where empty query_string is coming from
		96	delete $query->{query}->{query_string} if
		97	$query->{query}->{query_string} && !%{$query->{query}->{query_string}};
		98
94	my $results = eval {	99	my $results = eval {
95	$elasticsearch->search(	100	$elasticsearch->search(
96	index => $self->index_name,	101	index => $self->index_name,

Lines 41-46 search: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
41	ci_raw:	41	ci_raw:
42	type: keyword	42	type: keyword
43	normalizer: icu_folding_normalizer	43	normalizer: icu_folding_normalizer
		44	geo_point:
		45	type: geo_point
44	default:	46	default:
45	type: text	47	type: text
46	analyzer: analyzer_standard	48	analyzer: analyzer_standard




    opac: 1
    staff_client: 1
    type: ''
  geolocation_lat:
    label: geolocation_lat
    mappings:
      - facet: ''
        marc_field: 034s
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  geolocation_lon:
    label: geolocation_lon
    mappings:
      - facet: ''
        marc_field: 034t
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  holdingbranch:
    facet_order: 8
    label: holdinglibrary




use Modern::Perl;

return {
    bug_number  => "31652",
    description => "Add geo-search: new value for search_field.type enum",
    up          => sub {
        my ($args) = @_;
        my ( $dbh, $out ) = @$args{qw(dbh out)};

        # Do you stuffs here
        $dbh->do(
            q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') }
        );

        # Print useful stuff here
        say $out "Added new value 'geo_point' to search_field.type enum";
        $dbh->do(
            q{INSERT IGNORE INTO systempreferences ( 'variable', 'value', 'options', 'explanation', 'type' ) VALUES ('GeoSearchEnabled', '0', NULL, 'Enable GeoSearch Feature via Elasticsearch', 'YesNo')
        say $out "Added new system preference 'GeoSearchEnabled'";
12      }
        );
    },
};

Lines 5607-5615 DROP TABLE IF EXISTS `search_field`; Link Here

(-)a/installer/data/mysql/kohastructure.sql (-3 / +3 lines)
5607	/!40101 SET character_set_client = utf8 /;	5607	/!40101 SET character_set_client = utf8 /;
5608	CREATE TABLE `search_field` (	5608	CREATE TABLE `search_field` (
5609	`id` int(11) NOT NULL AUTO_INCREMENT,	5609	`id` int(11) NOT NULL AUTO_INCREMENT,
5610	`name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',	5610	`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
5611	`label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',	5611	`label` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the human readable name of the field, for display',
5612	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',	5612	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
5613	`weight` decimal(5,2) DEFAULT NULL,	5613	`weight` decimal(5,2) DEFAULT NULL,
5614	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',	5614	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',
5615	`staff_client` tinyint(1) NOT NULL DEFAULT 1,	5615	`staff_client` tinyint(1) NOT NULL DEFAULT 1,

Lines 99-104 Searching: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+7 lines)
99	- "List of search fields (separated by \| or ,) that should not be autotruncated by Elasticsearch even if <a href='/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=QueryAutoTruncate'>QueryAutoTruncate</a> is set to 'Yes':"	99	- "List of search fields (separated by \| or ,) that should not be autotruncated by Elasticsearch even if <a href='/cgi-bin/koha/admin/preferences.pl?op=search&searchfield=QueryAutoTruncate'>QueryAutoTruncate</a> is set to 'Yes':"
100	- pref: ESPreventAutoTruncate	100	- pref: ESPreventAutoTruncate
101	class: long	101	class: long
		102	-
		103	- pref: GeoSearchEnabled
		104	type: boolean
		105	choices:
		106	1: Enable
		107	0: Disable
		108	- 'GeoSearch via Elasticsearch'
102	Search form:	109	Search form:
103	-	110	-
104	- pref: LoadSearchHistoryToTheFirstLoggedUser	111	- pref: LoadSearchHistoryToTheFirstLoggedUser

Lines 222-227 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
222	[% ELSE %]	222	[% ELSE %]
223	<option value="callnumber">Call Number</option>	223	<option value="callnumber">Call Number</option>
224	[% END %]	224	[% END %]
		225	[% IF search_field.type == "geo_point" %]
		226	<option value="geo_point" selected="selected">Geo Point</option>
		227	[% ELSE %]
		228	<option value="geo_point">Geo Point</option>
		229	[% END %]
225	</select>	230	</select>
226	</td>	231	</td>
227	<td data-order="[% search_field.weight \| html %]">	232	<td data-order="[% search_field.weight \| html %]">




    is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');
};

subtest '_create_query_string' => sub {
    plan tests => 2;

    my $params = {
        index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX,
    };
    my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new($params);

    my @queries;
    my $normal_query = [
        {
            'operand'  => 'perl*',
            'operator' => undef
        }
    ];

    @queries = $qb->_create_query_string(@$normal_query);
    my $expect = ['(perl*)'];

    is( @queries, @$expect, 'expected search structure' );

    my $geo_query = [
        {
            'operator' => undef,
            'field'    => 'geolocation',
            'type'     => undef,
            'operand'  => 'lat:48.25* lng:16.35* distance:100km*'
        }
    ];

    @queries = $qb->_create_query_string(@$geo_query);
    my $expect_geo = [];
    is( @queries, @$expect_geo, 'expected geo search structure => empty normal search string' );

};


1;
- 

Return to bug 31652

Lines 1855-1860 biblios: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (+18 lines)
1855	opac: 1	1855	opac: 1
1856	staff_client: 1	1856	staff_client: 1
1857	type: ''	1857	type: ''
		1858	geolocation_lat:
		1859	label: geolocation_lat
		1860	mappings:
		1861	- facet: ''
		1862	marc_field: 034s
		1863	marc_type: marc21
		1864	sort: 0
		1865	suggestible: ''
		1866	type: geo_point
		1867	geolocation_lon:
		1868	label: geolocation_lon
		1869	mappings:
		1870	- facet: ''
		1871	marc_field: 034t
		1872	marc_type: marc21
		1873	sort: 0
		1874	suggestible: ''
		1875	type: geo_point
1858	holdingbranch:	1876	holdingbranch:
1859	facet_order: 8	1877	facet_order: 8
1860	label: holdinglibrary	1878	label: holdinglibrary

Lines 333-336 subtest '_join_queries' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-1 / +37 lines)
333	is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');	333	is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');
334	};	334	};
335		335
		336	subtest '_create_query_string' => sub {
		337	plan tests => 2;
		338
		339	my $params = {
		340	index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX,
		341	};
		342	my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new($params);
		343
		344	my @queries;
		345	my $normal_query = [
		346	{
		347	'operand' => 'perl*',
		348	'operator' => undef
		349	}
		350	];
		351
		352	@queries = $qb->_create_query_string(@$normal_query);
		353	my $expect = ['(perl*)'];
		354
		355	is( @queries, @$expect, 'expected search structure' );
		356
		357	my $geo_query = [
		358	{
		359	'operator' => undef,
		360	'field' => 'geolocation',
		361	'type' => undef,
		362	'operand' => 'lat:48.25* lng:16.35* distance:100km*'
		363	}
		364	];
		365
		366	@queries = $qb->_create_query_string(@$geo_query);
		367	my $expect_geo = [];
		368	is( @queries, @$expect_geo, 'expected geo search structure => empty normal search string' );
		369
		370	};
		371
		372
336	1;	373	1;
337	-

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_31652_add_geo_search.pl (+23 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "31652",
	5	description => "Add geo-search: new value for search_field.type enum",
	6	up => sub {
	7	my ($args) = @_;
	8	my ( $dbh, $out ) = @$args{qw(dbh out)};
	9
	10	# Do you stuffs here
	11	$dbh->do(
	12	q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') }
	13	);
	14
	15	# Print useful stuff here
	16	say $out "Added new value 'geo_point' to search_field.type enum";
	17	$dbh->do(
	18	q{INSERT IGNORE INTO systempreferences ( 'variable', 'value', 'options', 'explanation', 'type' ) VALUES ('GeoSearchEnabled', '0', NULL, 'Enable GeoSearch Feature via Elasticsearch', 'YesNo')
	19	say $out "Added new system preference 'GeoSearchEnabled'";
	20	12 }
	21	);
	22	},
	23	};