Attachment #141406 for bug #31652

View | Details | Raw Unified | Return to bug 31652
Collapse All | Expand All




=head2 type

  data_type: 'enum'
  extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber","geo_point"]}
  is_nullable: 0

what type of data this holds, relevant when storing it in the search engine

        "stdno",
        "year",
        "callnumber",
        "geo_point",
      ],
    },
    is_nullable => 0,

);


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-09-29 12:18:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xtsGkd3Gjnqw8ZDmoeRDzQ

__PACKAGE__->add_columns(
    '+mandatory' => { is_boolean => 1 },

Lines 212-217 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (+22 lines)
212	$es_type = 'year';	212	$es_type = 'year';
213	} elsif ($type eq 'callnumber') {	213	} elsif ($type eq 'callnumber') {
214	$es_type = 'cn_sort';	214	$es_type = 'cn_sort';
		215	} elsif ($type eq 'geo_point') {
		216	$es_type = 'geo_point';
		217	}
		218
		219	if ($type eq 'geo_point') {
		220	$name =~ s/_(lat\|lon)$//;
215	}	221	}
216		222
217	if ($search) {	223	if ($search) {
Lines 729-734 sub marc_records_to_documents { Link Here
729	}	735	}
730	}	736	}
731		737
		738	foreach my $field (@{$rules->{geo_point}}) {
		739	next unless $record_document->{$field};
		740	my $geofield = $field;
		741	$geofield =~ s/_(lat\|lon)$//;
		742	my $axis = $1;
		743	my $vals = $record_document->{$field};
		744	for my $i (0 .. @$vals - 1) {
		745	my $val = $record_document->{$field}[$i];
		746	$record_document->{$geofield}[$i]{$axis} = $val;
		747	}
		748	delete $record_document->{$field};
		749	}
		750
732	# Remove duplicate values and collapse sort fields	751	# Remove duplicate values and collapse sort fields
733	foreach my $field (keys %{$record_document}) {	752	foreach my $field (keys %{$record_document}) {
734	if (ref($record_document->{$field}) eq 'ARRAY') {	753	if (ref($record_document->{$field}) eq 'ARRAY') {
Lines 1052-1057 sub _get_marc_mapping_rules { Link Here
1052	elsif ($type eq 'isbn') {	1071	elsif ($type eq 'isbn') {
1053	push @{$rules->{isbn}}, $name;	1072	push @{$rules->{isbn}}, $name;
1054	}	1073	}
		1074	elsif ($type eq 'geo_point') {
		1075	push @{$rules->{geo_point}}, $name;
		1076	}
1055	elsif ($type eq 'boolean') {	1077	elsif ($type eq 'boolean') {
1056	# boolean gets special handling, if value doesn't exist for a field,	1078	# boolean gets special handling, if value doesn't exist for a field,
1057	# it is set to false	1079	# it is set to false

Lines 117-122 sub update_index { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +1 lines)
117	}	117	}
118		118
119	my $documents = $self->marc_records_to_documents($records);	119	my $documents = $self->marc_records_to_documents($records);
		120
120	my @body;	121	my @body;
121	for (my $i = 0; $i < scalar @$index_record_ids; $i++) {	122	for (my $i = 0; $i < scalar @$index_record_ids; $i++) {
122	my $id = $index_record_ids->[$i];	123	my $id = $index_record_ids->[$i];
Lines 266-272 sub update_mappings { Link Here
266	my ($self) = @_;	267	my ($self) = @_;
267	my $elasticsearch = $self->get_elasticsearch();	268	my $elasticsearch = $self->get_elasticsearch();
268	my $mappings = $self->get_elasticsearch_mappings();	269	my $mappings = $self->get_elasticsearch_mappings();
269
270	try {	270	try {
271	my $response = $elasticsearch->indices->put_mapping(	271	my $response = $elasticsearch->indices->put_mapping(
272	index => $self->index_name,	272	index => $self->index_name,

Lines 136-141 our %index_field_convert = ( Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +45 lines)
136	);	136	);
137	my $field_name_pattern = '[\w\-]+';	137	my $field_name_pattern = '[\w\-]+';
138	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";	138	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";
		139	my $es_advanced_searches = [];
139		140
140	=head2 get_index_field_convert	141	=head2 get_index_field_convert
141		142
Lines 244-249 sub build_query { Link Here
244	or $display_library_facets eq 'holding' ) {	245	or $display_library_facets eq 'holding' ) {
245	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };	246	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };
246	}	247	}
		248
		249	$res = _rebuild_to_es_advanced_query($res) if @$es_advanced_searches ;
247	return $res;	250	return $res;
248	}	251	}
249		252
Lines 908-913 operand. Link Here
908		911
909	sub _create_query_string {	912	sub _create_query_string {
910	my ( $self, @queries ) = @_;	913	my ( $self, @queries ) = @_;
		914	foreach my $q (@queries) {
		915	if ($q->{field} && $q->{field} eq 'geolocation') {
		916	push(@$es_advanced_searches, $q);
		917	}
		918	}
		919	@queries = grep { $_->{field} ne 'geolocation' } @queries;
911		920
912	map {	921	map {
913	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';	922	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';
Lines 1061-1067 sub _fix_limit_special_cases { Link Here
1061		1070
1062	my @new_lim;	1071	my @new_lim;
1063	foreach my $l (@$limits) {	1072	foreach my $l (@$limits) {
1064
1065	# This is set up by opac-search.pl	1073	# This is set up by opac-search.pl
1066	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {	1074	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {
1067	my ( $start, $end ) =	1075	my ( $start, $end ) =
Lines 1291-1294 sub _search_fields { Link Here
1291	}	1299	}
1292	}	1300	}
1293		1301
		1302	sub _rebuild_to_es_advanced_query {
		1303	my ($res) = @_;
		1304	my $query_string = $res->{query}->{query_string};
		1305	$query_string->{query} = '*' unless $query_string->{query};
		1306	delete $res->{query}->{query_string};
		1307
		1308	my %filter;
		1309	for my $advanced_query (@$es_advanced_searches) {
		1310	if ( $advanced_query->{field} eq 'geolocation') {
		1311	my ($lat, $lon, $distance) = map { $_ =~ /:(.)\/ } split('\s+', $advanced_query->{operand});
		1312	$filter{geo_distance} = {
		1313	distance => $distance,
		1314	geolocation => {
		1315	lat => $lat,
		1316	lon => $lon,
		1317	}
		1318	};
		1319	}
		1320	else {
		1321	warn "unknown advanced ElasticSearch query: ".join(', ',%$advanced_query);
		1322	}
		1323	}
		1324
		1325	$res->{query} = {
		1326	bool => {
		1327	must => {
		1328	query_string => $query_string
		1329	},
		1330	filter => \%filter,
		1331	}
		1332	};
		1333
		1334	return $res;
		1335	}
		1336
		1337
1294	1;	1338	1;

Lines 91-96 sub search { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+5 lines)
91	$query->{from} = $page * $query->{size};	91	$query->{from} = $page * $query->{size};
92	}	92	}
93	my $elasticsearch = $self->get_elasticsearch();	93	my $elasticsearch = $self->get_elasticsearch();
		94
		95	# XXX investigate where empty query_string is coming from
		96	delete $query->{query}->{query_string} if
		97	$query->{query}->{query_string} && !%{$query->{query}->{query_string}};
		98
94	my $results = eval {	99	my $results = eval {
95	$elasticsearch->search(	100	$elasticsearch->search(
96	index => $self->index_name,	101	index => $self->index_name,

Lines 38-43 search: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
38	search_analyzer: analyzer_phrase	38	search_analyzer: analyzer_phrase
39	raw:	39	raw:
40	type: keyword	40	type: keyword
		41	geo_point:
		42	type: geo_point
41	default:	43	default:
42	type: text	44	type: text
43	analyzer: analyzer_standard	45	analyzer: analyzer_standard




        sort: ~
        suggestible: ''
    type: ''
  geolocation_lat:
    label: geolocation_lat
    mappings:
      - facet: ''
        marc_field: 034s
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  geolocation_lon:
    label: geolocation_lon
    mappings:
      - facet: ''
        marc_field: 034t
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  holdingbranch:
    facet_order: 8
    label: holdinglibrary




use Modern::Perl;

return {
    bug_number => "31652",
    description => "Add geo-search: new value for search_field.type enum",
    up => sub {
        my ($args) = @_;
        my ($dbh, $out) = @$args{qw(dbh out)};
        # Do you stuffs here
        $dbh->do(q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') });
        # Print useful stuff here
        say $out "Added new value 'geo_point' to search_field.type enum";
    },
};

Lines 4709-4715 CREATE TABLE `search_field` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (-1 / +1 lines)
4709	`id` int(11) NOT NULL AUTO_INCREMENT,	4709	`id` int(11) NOT NULL AUTO_INCREMENT,
4710	`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',	4710	`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
4711	`label` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the human readable name of the field, for display',	4711	`label` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'the human readable name of the field, for display',
4712	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber') COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',	4712	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
4713	`weight` decimal(5,2) DEFAULT NULL,	4713	`weight` decimal(5,2) DEFAULT NULL,
4714	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',	4714	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',
4715	`staff_client` tinyint(1) NOT NULL DEFAULT 1,	4715	`staff_client` tinyint(1) NOT NULL DEFAULT 1,

Lines 224-229 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
224	[% ELSE %]	224	[% ELSE %]
225	<option value="callnumber">Call Number</option>	225	<option value="callnumber">Call Number</option>
226	[% END %]	226	[% END %]
		227	[% IF search_field.type == "geo_point" %]
		228	<option value="geo_point" selected="selected">Geo Point</option>
		229	[% ELSE %]
		230	<option value="geo_point">Geo Point</option>
		231	[% END %]
227	</select>	232	</select>
228	</td>	233	</td>
229	<td data-order="[% search_field.weight \| html %]">	234	<td data-order="[% search_field.weight \| html %]">

Lines 106-112 my $format = $cgi->param("format") \|\| ''; Link Here

(-)a/opac/opac-search.pl (-4 / +18 lines)
106	if ($format =~ /(rss\|atom\|opensearchdescription)/) {	106	if ($format =~ /(rss\|atom\|opensearchdescription)/) {
107	$template_name = 'opac-opensearch.tt';	107	$template_name = 'opac-opensearch.tt';
108	}	108	}
109	elsif ((@params>=1) \|\| (defined $cgi->param("q") && $cgi->param("q") ne "") \|\| ($cgi->param('multibranchlimit')) \|\| ($cgi->param('limit-yr')) \|\| @searchCategories ) {	109	elsif ((@params>=1) \|\| (defined $cgi->param("q") && $cgi->param("q") ne "") \|\|
		110	($cgi->param('multibranchlimit')) \|\| ($cgi->param('limit-yr')) \|\| @searchCategories
		111	\|\| $cgi->param('lat') ){
110	$template_name = 'opac-results.tt';	112	$template_name = 'opac-results.tt';
111	}	113	}
112	else {	114	else {
Lines 402-409 my @operators = $cgi->multi_param('op'); Link Here
402		404
403	# indexes are query qualifiers, like 'title', 'author', etc. They	405	# indexes are query qualifiers, like 'title', 'author', etc. They
404	# can be single or multiple parameters separated by comma: kw,right-Truncation	406	# can be single or multiple parameters separated by comma: kw,right-Truncation
		407
		408	if ($params->{'lat'} && $params->{'lng'} && $params->{'distance'} ) {
		409	$params->{q} = sprintf("lat:%s lng:%s distance:%s",
		410	$params->{'lat'}, $params->{'lng'}, $params->{'distance'});
		411
		412	$params->{idx} = 'geolocation';
		413	delete $params->{'lat'};
		414	delete $params->{'lng'};
		415	delete $params->{'radius'};
		416	}
		417
		418
		419
405	my @indexes = $cgi->multi_param('idx');	420	my @indexes = $cgi->multi_param('idx');
406	@indexes = map { uri_unescape($_) } @indexes;	421	@indexes = grep { $_ } @indexes;
		422	@indexes = map { uri_unescape($_) } @indexes;
407		423
408	# if a simple index (only one) display the index used in the top search box	424	# if a simple index (only one) display the index used in the top search box
409	if ($indexes[0] && !$indexes[1]) {	425	if ($indexes[0] && !$indexes[1]) {
Lines 457-463 foreach my $limit(@limits) { Link Here
457	}	473	}
458	$template->param(available => $available);	474	$template->param(available => $available);
459		475
460	# append year limits if they exist
461	if ($params->{'limit-yr'}) {	476	if ($params->{'limit-yr'}) {
462	if ($params->{'limit-yr'} =~ /\d{4}/) {	477	if ($params->{'limit-yr'} =~ /\d{4}/) {
463	push @limits, "yr,st-numeric=$params->{'limit-yr'}";	478	push @limits, "yr,st-numeric=$params->{'limit-yr'}";
464	-

Return to bug 31652

Lines 1555-1560 biblios: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (+18 lines)
1555	sort: ~	1555	sort: ~
1556	suggestible: ''	1556	suggestible: ''
1557	type: ''	1557	type: ''
		1558	geolocation_lat:
		1559	label: geolocation_lat
		1560	mappings:
		1561	- facet: ''
		1562	marc_field: 034s
		1563	marc_type: marc21
		1564	sort: 0
		1565	suggestible: ''
		1566	type: geo_point
		1567	geolocation_lon:
		1568	label: geolocation_lon
		1569	mappings:
		1570	- facet: ''
		1571	marc_field: 034t
		1572	marc_type: marc21
		1573	sort: 0
		1574	suggestible: ''
		1575	type: geo_point
1558	holdingbranch:	1576	holdingbranch:
1559	facet_order: 8	1577	facet_order: 8
1560	label: holdinglibrary	1578	label: holdinglibrary

Lines 48-54 the human readable name of the field, for display Link Here

(-)a/Koha/Schema/Result/SearchField.pm (-3 / +4 lines)
48	=head2 type	48	=head2 type
49		49
50	data_type: 'enum'	50	data_type: 'enum'
51	extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber"]}	51	extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber","geo_point"]}
52	is_nullable: 0	52	is_nullable: 0
53		53
54	what type of data this holds, relevant when storing it in the search engine	54	what type of data this holds, relevant when storing it in the search engine
Lines 109-114 __PACKAGE__->add_columns( Link Here
109	"stdno",	109	"stdno",
110	"year",	110	"year",
111	"callnumber",	111	"callnumber",
		112	"geo_point",
112	],	113	],
113	},	114	},
114	is_nullable => 0,	115	is_nullable => 0,
Lines 169-176 __PACKAGE__->has_many( Link Here
169	);	170	);
170		171
171		172
172	# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-07-18 15:10:43	173	# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-09-29 12:18:11
173	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uk5JsfPJo0XVvGfMfJg3cg	174	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xtsGkd3Gjnqw8ZDmoeRDzQ
174		175
175	__PACKAGE__->add_columns(	176	__PACKAGE__->add_columns(
176	'+mandatory' => { is_boolean => 1 },	177	'+mandatory' => { is_boolean => 1 },

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_31652_add_geo_search.pl (+14 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "31652",
	5	description => "Add geo-search: new value for search_field.type enum",
	6	up => sub {
	7	my ($args) = @_;
	8	my ($dbh, $out) = @$args{qw(dbh out)};
	9	# Do you stuffs here
	10	$dbh->do(q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') });
	11	# Print useful stuff here
	12	say $out "Added new value 'geo_point' to search_field.type enum";
	13	},
	14	};