From 3a97e7a554565206628513cb92b4a38e2f26c72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20=E2=80=9Etadzik=E2=80=9D=20So=C5=9Bnierz?= Date: Mon, 23 Sep 2024 13:39:43 +0200 Subject: [PATCH] Bug 37985: Extend geo-search to support areas, not just points (MARC 034 subfields defg) --- Koha/SearchEngine/Elasticsearch.pm | 38 +++++++++- .../Elasticsearch/QueryBuilder.pm | 72 +++++++++++++++---- .../elasticsearch/field_config.yaml | 2 + .../searchengine/elasticsearch/mappings.yaml | 36 ++++++++++ installer/data/mysql/kohastructure.sql | 2 +- .../SearchEngine/Elasticsearch/QueryBuilder.t | 24 ++++++- 6 files changed, 157 insertions(+), 17 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 5ef5235720..51d2aa4d85 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -215,10 +215,10 @@ sub get_elasticsearch_mappings { $es_type = 'cn_sort'; } elsif ($type eq 'geo_point') { $es_type = 'geo_point'; - } - - if ($type eq 'geo_point') { $name =~ s/_(lat|lon)$//; + } elsif ($type eq 'geo_shape') { + $es_type = 'geo_shape'; + $name =~ s/_(north|south|east|west)ernmost$/_area/; } if ($search) { @@ -789,6 +789,35 @@ sub marc_records_to_documents { delete $record_document->{$field}; } + my %shape_bounds; + foreach my $field ( @{ $rules->{geo_shape} } ) { + next unless $record_document->{$field}; + $field =~ /([^_]+)_(north|south|east|west)ernmost$/; + my $geofield = $1; + my $direction = $2; + $shape_bounds{$geofield}{$direction} = $record_document->{$field}; + delete $record_document->{$field}; + } + for my $geofield (keys %shape_bounds) { + my %bounds = %{$shape_bounds{$geofield}}; + if (%bounds == 4) { + my @shapes; + for my $i (0..scalar($bounds{north}->@*) - 1) { + $shapes[$i] = [ + [ 0+$bounds{west}[$i], 0+$bounds{north}[$i] ], + [ 0+$bounds{east}[$i], 0+$bounds{south}[$i] ], + ]; + } + my $es_field = $geofield . '_area'; + $record_document->{$es_field} = [map { + +{ + type => 'envelope', + coordinates => $_, + } + } @shapes]; + } + } + # Remove duplicate values and collapse sort fields foreach my $field (keys %{$record_document}) { if (ref($record_document->{$field}) eq 'ARRAY') { @@ -1127,6 +1156,9 @@ sub _get_marc_mapping_rules { elsif ($type eq 'geo_point') { push @{$rules->{geo_point}}, $name; } + elsif ($type eq 'geo_shape') { + push @{$rules->{geo_shape}}, $name; + } elsif ($type eq 'boolean') { # boolean gets special handling, if value doesn't exist for a field, # it is set to false diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index bd0ba28197..6e02c64fc8 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -1414,20 +1414,66 @@ sub _is_safe_to_auto_truncate { sub _rebuild_to_es_advanced_query { my ($res) = @_; my $query_string = $res->{query}->{query_string}; - $query_string->{query} = '*' unless $query_string->{query}; delete $res->{query}->{query_string}; - my %filter; + my @should; for my $advanced_query (@$es_advanced_searches) { if ( $advanced_query->{field} eq 'geolocation' ) { - my ( $lat, $lon, $distance ) = map { $_ =~ /:(.*)\*/ } split( '\s+', $advanced_query->{operand} ); - $filter{geo_distance} = { - distance => $distance, - geolocation => { - lat => $lat, - lon => $lon, - } - }; + my %args = map { split ':' } map { s/\*$//r } split /\s+/, $advanced_query->{operand}; + if ($args{lat} and $args{lng} and $args{distance}) { + push @should, + { + geo_distance => { + distance => $args{distance}, + geolocation => { + lat => $args{lat}, + lon => $args{lng}, + } + } + }, + { + geo_distance => { + distance => $args{distance}, + geolocation_area => { + lat => $args{lat}, + lon => $args{lng}, + } + } + }; + } elsif ($args{boundingbox}) { + my ($top_left_lat, $top_left_lon, $bottom_right_lat, $bottom_right_lon) = split ',', $args{boundingbox}; + push @should, + { + geo_bounding_box => { + 'geolocation' => { + 'top_left' => { + 'lat' => $top_left_lat, + 'lon' => $top_left_lon, + }, + 'bottom_right' => { + 'lat' => $bottom_right_lat, + 'lon' => $bottom_right_lon, + } + }, + } + }, + { + geo_bounding_box => { + 'geolocation_area' => { + 'top_left' => { + 'lat' => $top_left_lat, + 'lon' => $top_left_lon, + }, + 'bottom_right' => { + 'lat' => $bottom_right_lat, + 'lon' => $bottom_right_lon, + } + }, + } + }; + } else { + warn "Unrecognized parameter set for geolocation queries: " . join(', ', keys %args); + } } else { warn "unknown advanced ElasticSearch query: " . join( ', ', %$advanced_query ); } @@ -1435,10 +1481,12 @@ sub _rebuild_to_es_advanced_query { $res->{query} = { bool => { - must => { query_string => $query_string }, - filter => \%filter, + should => \@should, } }; + if ($query_string->{query}) { + $res->{query}{bool}{must} = { query_string => $query_string }; + } return $res; } diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index 41adcef7ed..eb3d2ff81d 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -43,6 +43,8 @@ search: normalizer: icu_folding_normalizer geo_point: type: geo_point + geo_shape: + type: geo_shape default: type: text analyzer: analyzer_standard diff --git a/admin/searchengine/elasticsearch/mappings.yaml b/admin/searchengine/elasticsearch/mappings.yaml index 17e513fb6f..da7ec0f1e6 100644 --- a/admin/searchengine/elasticsearch/mappings.yaml +++ b/admin/searchengine/elasticsearch/mappings.yaml @@ -1916,6 +1916,42 @@ biblios: opac: 1 staff_client: 1 type: '' + geolocation_westernmost: + label: geolocation_westernmost + mappings: + - facet: '' + marc_field: 034d + marc_type: marc21 + sort: 0 + suggestible: '' + type: geo_shape + geolocation_easternmost: + label: geolocation_easternmost + mappings: + - facet: '' + marc_field: 034e + marc_type: marc21 + sort: 0 + suggestible: '' + type: geo_shape + geolocation_northernmost: + label: geolocation_northernmost + mappings: + - facet: '' + marc_field: 034f + marc_type: marc21 + sort: 0 + suggestible: '' + type: geo_shape + geolocation_southernmost: + label: geolocation_southernmost + mappings: + - facet: '' + marc_field: 034g + marc_type: marc21 + sort: 0 + suggestible: '' + type: geo_shape geolocation_lat: label: geolocation_lat mappings: diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 2b2175fe2f..74a1e3f125 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -5660,7 +5660,7 @@ CREATE TABLE `search_field` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine', `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display', - `type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', + `type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point','geo_shape') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', `weight` decimal(5,2) DEFAULT NULL, `facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted', `staff_client` tinyint(1) NOT NULL DEFAULT 1, diff --git a/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t b/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t index b6b670d7d5..fc39107ae7 100755 --- a/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t +++ b/t/Koha/SearchEngine/Elasticsearch/QueryBuilder.t @@ -17,7 +17,7 @@ use Modern::Perl; -use Test::More tests => 8; +use Test::More tests => 9; use t::lib::Mocks; use_ok('Koha::SearchEngine::Elasticsearch::QueryBuilder'); @@ -369,5 +369,27 @@ subtest '_create_query_string' => sub { }; +subtest 'geolocation queries' => sub { + my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ + index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, + }); + + $qb->_create_query_string({ + 'field' => 'geolocation', + 'operand' => 'lat:48.25* lng:16.35* distance:100km*' + }); + my $query = $qb->build_query(); + is_deeply $query->{query}{bool}{should}, [ + { geo_distance => { + distance => '100km', + geolocation => { lat => 48.25, lon => 16.35 }, + } }, + { geo_distance => { + distance => '100km', + geolocation_area => { lat => 48.25, lon => 16.35 }, + } }, + ], 'should be able to create "distance from a point" queries'; + +}; 1; -- 2.45.2