Attachment #164666 for bug #31652

View | Details | Raw Unified | Return to bug 31652
Collapse All | Expand All

Lines 212-217 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (+22 lines)
212	$es_type = 'year';	212	$es_type = 'year';
213	} elsif ($type eq 'callnumber') {	213	} elsif ($type eq 'callnumber') {
214	$es_type = 'cn_sort';	214	$es_type = 'cn_sort';
		215	} elsif ($type eq 'geo_point') {
		216	$es_type = 'geo_point';
		217	}
		218
		219	if ($type eq 'geo_point') {
		220	$name =~ s/_(lat\|lon)$//;
215	}	221	}
216		222
217	if ($search) {	223	if ($search) {
Lines 735-740 sub marc_records_to_documents { Link Here
735	}	741	}
736	}	742	}
737		743
		744	foreach my $field (@{$rules->{geo_point}}) {
		745	next unless $record_document->{$field};
		746	my $geofield = $field;
		747	$geofield =~ s/_(lat\|lon)$//;
		748	my $axis = $1;
		749	my $vals = $record_document->{$field};
		750	for my $i (0 .. @$vals - 1) {
		751	my $val = $record_document->{$field}[$i];
		752	$record_document->{$geofield}[$i]{$axis} = $val;
		753	}
		754	delete $record_document->{$field};
		755	}
		756
738	# Remove duplicate values and collapse sort fields	757	# Remove duplicate values and collapse sort fields
739	foreach my $field (keys %{$record_document}) {	758	foreach my $field (keys %{$record_document}) {
740	if (ref($record_document->{$field}) eq 'ARRAY') {	759	if (ref($record_document->{$field}) eq 'ARRAY') {
Lines 1070-1075 sub _get_marc_mapping_rules { Link Here
1070	elsif ($type eq 'isbn') {	1089	elsif ($type eq 'isbn') {
1071	push @{$rules->{isbn}}, $name;	1090	push @{$rules->{isbn}}, $name;
1072	}	1091	}
		1092	elsif ($type eq 'geo_point') {
		1093	push @{$rules->{geo_point}}, $name;
		1094	}
1073	elsif ($type eq 'boolean') {	1095	elsif ($type eq 'boolean') {
1074	# boolean gets special handling, if value doesn't exist for a field,	1096	# boolean gets special handling, if value doesn't exist for a field,
1075	# it is set to false	1097	# it is set to false

Lines 137-142 our %index_field_convert = ( Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +50 lines)
137	);	137	);
138	my $field_name_pattern = '[\w\-]+';	138	my $field_name_pattern = '[\w\-]+';
139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";	139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";
		140	my $es_advanced_searches = [];
140		141
141	=head2 get_index_field_convert	142	=head2 get_index_field_convert
142		143
Lines 245-250 sub build_query { Link Here
245	or $display_library_facets eq 'holding' ) {	246	or $display_library_facets eq 'holding' ) {
246	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };	247	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };
247	}	248	}
		249
		250	$res = _rebuild_to_es_advanced_query($res) if @$es_advanced_searches ;
248	return $res;	251	return $res;
249	}	252	}
250		253
Lines 929-934 operand. Link Here
929		932
930	sub _create_query_string {	933	sub _create_query_string {
931	my ( $self, @queries ) = @_;	934	my ( $self, @queries ) = @_;
		935	$es_advanced_searches = [];
		936	my @string_queries;
		937	foreach my $q (@queries) {
		938	if ($q->{field} && $q->{field} eq 'geolocation') {
		939	push(@$es_advanced_searches, $q);
		940	} else {
		941	push(@string_queries, $q)
		942	}
		943	}
		944
		945	@queries = @string_queries;
932		946
933	map {	947	map {
934	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';	948	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';
Lines 1082-1088 sub _fix_limit_special_cases { Link Here
1082		1096
1083	my @new_lim;	1097	my @new_lim;
1084	foreach my $l (@$limits) {	1098	foreach my $l (@$limits) {
1085
1086	# This is set up by opac-search.pl	1099	# This is set up by opac-search.pl
1087	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {	1100	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {
1088	my ( $start, $end ) =	1101	my ( $start, $end ) =
Lines 1338-1341 sub _search_fields { Link Here
1338	}	1351	}
1339	}	1352	}
1340		1353
		1354	sub _rebuild_to_es_advanced_query {
		1355	my ($res) = @_;
		1356	my $query_string = $res->{query}->{query_string};
		1357	$query_string->{query} = '*' unless $query_string->{query};
		1358	delete $res->{query}->{query_string};
		1359
		1360	my %filter;
		1361	for my $advanced_query (@$es_advanced_searches) {
		1362	if ( $advanced_query->{field} eq 'geolocation') {
		1363	my ($lat, $lon, $distance) = map { $_ =~ /:(.)\/ } split('\s+', $advanced_query->{operand});
		1364	$filter{geo_distance} = {
		1365	distance => $distance,
		1366	geolocation => {
		1367	lat => $lat,
		1368	lon => $lon,
		1369	}
		1370	};
		1371	}
		1372	else {
		1373	warn "unknown advanced ElasticSearch query: ".join(', ',%$advanced_query);
		1374	}
		1375	}
		1376
		1377	$res->{query} = {
		1378	bool => {
		1379	must => {
		1380	query_string => $query_string
		1381	},
		1382	filter => \%filter,
		1383	}
		1384	};
		1385
		1386	return $res;
		1387	}
		1388
		1389
1341	1;	1390	1;

Lines 91-96 sub search { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+5 lines)
91	$query->{from} = $page * $query->{size};	91	$query->{from} = $page * $query->{size};
92	}	92	}
93	my $elasticsearch = $self->get_elasticsearch();	93	my $elasticsearch = $self->get_elasticsearch();
		94
		95	# FixMe: investigate where empty query_string is coming from
		96	delete $query->{query}->{query_string} if
		97	$query->{query}->{query_string} && !%{$query->{query}->{query_string}};
		98
94	my $results = eval {	99	my $results = eval {
95	$elasticsearch->search(	100	$elasticsearch->search(
96	index => $self->index_name,	101	index => $self->index_name,

Lines 41-46 search: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
41	ci_raw:	41	ci_raw:
42	type: keyword	42	type: keyword
43	normalizer: icu_folding_normalizer	43	normalizer: icu_folding_normalizer
		44	geo_point:
		45	type: geo_point
44	default:	46	default:
45	type: text	47	type: text
46	analyzer: analyzer_standard	48	analyzer: analyzer_standard




    opac: 1
    staff_client: 1
    type: ''
  geolocation_lat:
    label: geolocation_lat
    mappings:
      - facet: ''
        marc_field: 034s
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  geolocation_lon:
    label: geolocation_lon
    mappings:
      - facet: ''
        marc_field: 034t
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  holdingbranch:
    facet_order: 8
    label: holdinglibrary




use Modern::Perl;

return {
    bug_number  => "31652",
    description => "Add geo-search: new value for search_field.type enum",
    up          => sub {
        my ($args) = @_;
        my ( $dbh, $out ) = @$args{qw(dbh out)};

        # Do you stuffs here
        $dbh->do(
            q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') }
        );

        # Print useful stuff here
        say $out "Added new value 'geo_point' to search_field.type enum";
        $dbh->do(
            q{INSERT IGNORE INTO systempreferences ( 'variable', 'value', 'options', 'explanation', 'type' ) VALUES ('GeoSearchEnabled', '0', NULL, 'Enable GeoSearch Feature via Elasticsearch', 'YesNo')
        say $out "Added new system preference 'GeoSearchEnabled'";
12      }
        );
    },
};

Lines 5603-5611 DROP TABLE IF EXISTS `search_field`; Link Here

(-)a/installer/data/mysql/kohastructure.sql (-3 / +3 lines)
5603	/!40101 SET character_set_client = utf8 /;	5603	/!40101 SET character_set_client = utf8 /;
5604	CREATE TABLE `search_field` (	5604	CREATE TABLE `search_field` (
5605	`id` int(11) NOT NULL AUTO_INCREMENT,	5605	`id` int(11) NOT NULL AUTO_INCREMENT,
5606	`name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',	5606	`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
5607	`label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',	5607	`label` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the human readable name of the field, for display',
5608	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',	5608	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
5609	`weight` decimal(5,2) DEFAULT NULL,	5609	`weight` decimal(5,2) DEFAULT NULL,
5610	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',	5610	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',
5611	`staff_client` tinyint(1) NOT NULL DEFAULT 1,	5611	`staff_client` tinyint(1) NOT NULL DEFAULT 1,

Lines 94-99 Searching: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref (+7 lines)
94	1: Enable	94	1: Enable
95	0: Disable	95	0: Disable
96	- "the option for staff with permission to create/edit custom saved search filters."	96	- "the option for staff with permission to create/edit custom saved search filters."
		97	-
		98	- pref: GeoSearchEnabled
		99	type: boolean
		100	choices:
		101	1: Enable
		102	0: Disable
		103	- 'GeoSearch via Elasticsearch'
97	Search form:	104	Search form:
98	-	105	-
99	- pref: LoadSearchHistoryToTheFirstLoggedUser	106	- pref: LoadSearchHistoryToTheFirstLoggedUser

Lines 222-227 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
222	[% ELSE %]	222	[% ELSE %]
223	<option value="callnumber">Call Number</option>	223	<option value="callnumber">Call Number</option>
224	[% END %]	224	[% END %]
		225	[% IF search_field.type == "geo_point" %]
		226	<option value="geo_point" selected="selected">Geo Point</option>
		227	[% ELSE %]
		228	<option value="geo_point">Geo Point</option>
		229	[% END %]
225	</select>	230	</select>
226	</td>	231	</td>
227	<td data-order="[% search_field.weight \| html %]">	232	<td data-order="[% search_field.weight \| html %]">





use Modern::Perl;

use Test::More tests => 7;
use t::lib::Mocks;

use_ok('Koha::SearchEngine::Elasticsearch::QueryBuilder');

    is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');
};

subtest '_create_query_string' => sub {
    plan tests => 2;

    my $params = {
        index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX,
    };
    my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new($params);

    my @queries;
    my $normal_query = [
        {
            'operand'  => 'perl*',
            'operator' => undef
        }
    ];

    @queries = $qb->_create_query_string(@$normal_query);
    my $expect = ['(perl*)'];

    is( @queries, @$expect, 'expected search structure' );

    my $geo_query = [
        {
            'operator' => undef,
            'field'    => 'geolocation',
            'type'     => undef,
            'operand'  => 'lat:48.25* lng:16.35* distance:100km*'
        }
    ];

    @queries = $qb->_create_query_string(@$geo_query);
    my $expect_geo = [];
    is( @queries, @$expect_geo, 'expected geo search structure => empty normal search string' );

};


1;
- 

Return to bug 31652

Lines 1855-1860 biblios: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (+18 lines)
1855	opac: 1	1855	opac: 1
1856	staff_client: 1	1856	staff_client: 1
1857	type: ''	1857	type: ''
		1858	geolocation_lat:
		1859	label: geolocation_lat
		1860	mappings:
		1861	- facet: ''
		1862	marc_field: 034s
		1863	marc_type: marc21
		1864	sort: 0
		1865	suggestible: ''
		1866	type: geo_point
		1867	geolocation_lon:
		1868	label: geolocation_lon
		1869	mappings:
		1870	- facet: ''
		1871	marc_field: 034t
		1872	marc_type: marc21
		1873	sort: 0
		1874	suggestible: ''
		1875	type: geo_point
1858	holdingbranch:	1876	holdingbranch:
1859	facet_order: 8	1877	facet_order: 8
1860	label: holdinglibrary	1878	label: holdinglibrary

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_31652_add_geo_search.pl (+23 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "31652",
	5	description => "Add geo-search: new value for search_field.type enum",
	6	up => sub {
	7	my ($args) = @_;
	8	my ( $dbh, $out ) = @$args{qw(dbh out)};
	9
	10	# Do you stuffs here
	11	$dbh->do(
	12	q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') }
	13	);
	14
	15	# Print useful stuff here
	16	say $out "Added new value 'geo_point' to search_field.type enum";
	17	$dbh->do(
	18	q{INSERT IGNORE INTO systempreferences ( 'variable', 'value', 'options', 'explanation', 'type' ) VALUES ('GeoSearchEnabled', '0', NULL, 'Enable GeoSearch Feature via Elasticsearch', 'YesNo')
	19	say $out "Added new system preference 'GeoSearchEnabled'";
	20	12 }
	21	);
	22	},
	23	};

Lines 17-23 Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-2 / +38 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19		19
20	use Test::More tests => 6;	20	use Test::More tests => 7;
21	use t::lib::Mocks;	21	use t::lib::Mocks;
22		22
23	use_ok('Koha::SearchEngine::Elasticsearch::QueryBuilder');	23	use_ok('Koha::SearchEngine::Elasticsearch::QueryBuilder');
Lines 303-306 subtest '_join_queries' => sub { Link Here
303	is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');	303	is($query, '(homebranch:foo) AND itype:(BOOK OR EBOOK) AND location:(SHELF)', 'should join "mc-" parts with AND if not the same field');
304	};	304	};
305		305
		306	subtest '_create_query_string' => sub {
		307	plan tests => 2;
		308
		309	my $params = {
		310	index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX,
		311	};
		312	my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new($params);
		313
		314	my @queries;
		315	my $normal_query = [
		316	{
		317	'operand' => 'perl*',
		318	'operator' => undef
		319	}
		320	];
		321
		322	@queries = $qb->_create_query_string(@$normal_query);
		323	my $expect = ['(perl*)'];
		324
		325	is( @queries, @$expect, 'expected search structure' );
		326
		327	my $geo_query = [
		328	{
		329	'operator' => undef,
		330	'field' => 'geolocation',
		331	'type' => undef,
		332	'operand' => 'lat:48.25* lng:16.35* distance:100km*'
		333	}
		334	];
		335
		336	@queries = $qb->_create_query_string(@$geo_query);
		337	my $expect_geo = [];
		338	is( @queries, @$expect_geo, 'expected geo search structure => empty normal search string' );
		339
		340	};
		341
		342
306	1;	343	1;
307	-