Attachment #153720 for bug #31652

View | Details | Raw Unified | Return to bug 31652
Collapse All | Expand All




=head2 type

  data_type: 'enum'
  extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber","geo_point"]}
  is_nullable: 0

what type of data this holds, relevant when storing it in the search engine

        "stdno",
        "year",
        "callnumber",
        "geo_point",
      ],
    },
    is_nullable => 0,

);


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-09-29 12:18:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xtsGkd3Gjnqw8ZDmoeRDzQ

__PACKAGE__->add_columns(
    '+mandatory' => { is_boolean => 1 },

Lines 212-217 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (+22 lines)
212	$es_type = 'year';	212	$es_type = 'year';
213	} elsif ($type eq 'callnumber') {	213	} elsif ($type eq 'callnumber') {
214	$es_type = 'cn_sort';	214	$es_type = 'cn_sort';
		215	} elsif ($type eq 'geo_point') {
		216	$es_type = 'geo_point';
		217	}
		218
		219	if ($type eq 'geo_point') {
		220	$name =~ s/_(lat\|lon)$//;
215	}	221	}
216		222
217	if ($search) {	223	if ($search) {
Lines 733-738 sub marc_records_to_documents { Link Here
733	}	739	}
734	}	740	}
735		741
		742	foreach my $field (@{$rules->{geo_point}}) {
		743	next unless $record_document->{$field};
		744	my $geofield = $field;
		745	$geofield =~ s/_(lat\|lon)$//;
		746	my $axis = $1;
		747	my $vals = $record_document->{$field};
		748	for my $i (0 .. @$vals - 1) {
		749	my $val = $record_document->{$field}[$i];
		750	$record_document->{$geofield}[$i]{$axis} = $val;
		751	}
		752	delete $record_document->{$field};
		753	}
		754
736	# Remove duplicate values and collapse sort fields	755	# Remove duplicate values and collapse sort fields
737	foreach my $field (keys %{$record_document}) {	756	foreach my $field (keys %{$record_document}) {
738	if (ref($record_document->{$field}) eq 'ARRAY') {	757	if (ref($record_document->{$field}) eq 'ARRAY') {
Lines 1056-1061 sub _get_marc_mapping_rules { Link Here
1056	elsif ($type eq 'isbn') {	1075	elsif ($type eq 'isbn') {
1057	push @{$rules->{isbn}}, $name;	1076	push @{$rules->{isbn}}, $name;
1058	}	1077	}
		1078	elsif ($type eq 'geo_point') {
		1079	push @{$rules->{geo_point}}, $name;
		1080	}
1059	elsif ($type eq 'boolean') {	1081	elsif ($type eq 'boolean') {
1060	# boolean gets special handling, if value doesn't exist for a field,	1082	# boolean gets special handling, if value doesn't exist for a field,
1061	# it is set to false	1083	# it is set to false

Lines 117-122 sub update_index { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +1 lines)
117	}	117	}
118		118
119	my $documents = $self->marc_records_to_documents($records);	119	my $documents = $self->marc_records_to_documents($records);
		120
120	my @body;	121	my @body;
121	for (my $i = 0; $i < scalar @$index_record_ids; $i++) {	122	for (my $i = 0; $i < scalar @$index_record_ids; $i++) {
122	my $id = $index_record_ids->[$i];	123	my $id = $index_record_ids->[$i];
Lines 266-272 sub update_mappings { Link Here
266	my ($self) = @_;	267	my ($self) = @_;
267	my $elasticsearch = $self->get_elasticsearch();	268	my $elasticsearch = $self->get_elasticsearch();
268	my $mappings = $self->get_elasticsearch_mappings();	269	my $mappings = $self->get_elasticsearch_mappings();
269
270	try {	270	try {
271	my $response = $elasticsearch->indices->put_mapping(	271	my $response = $elasticsearch->indices->put_mapping(
272	index => $self->index_name,	272	index => $self->index_name,

Lines 137-142 our %index_field_convert = ( Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-1 / +45 lines)
137	);	137	);
138	my $field_name_pattern = '[\w\-]+';	138	my $field_name_pattern = '[\w\-]+';
139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";	139	my $multi_field_pattern = "(?:\\.$field_name_pattern)*";
		140	my $es_advanced_searches = [];
140		141
141	=head2 get_index_field_convert	142	=head2 get_index_field_convert
142		143
Lines 245-250 sub build_query { Link Here
245	or $display_library_facets eq 'holding' ) {	246	or $display_library_facets eq 'holding' ) {
246	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };	247	$res->{aggregations}{holdingbranch} = { terms => { field => "holdingbranch__facet", size => $size } };
247	}	248	}
		249
		250	$res = _rebuild_to_es_advanced_query($res) if @$es_advanced_searches ;
248	return $res;	251	return $res;
249	}	252	}
250		253
Lines 929-934 operand. Link Here
929		932
930	sub _create_query_string {	933	sub _create_query_string {
931	my ( $self, @queries ) = @_;	934	my ( $self, @queries ) = @_;
		935	foreach my $q (@queries) {
		936	if ($q->{field} && $q->{field} eq 'geolocation') {
		937	push(@$es_advanced_searches, $q);
		938	}
		939	}
		940	@queries = grep { $_->{field} ne 'geolocation' } @queries;
932		941
933	map {	942	map {
934	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';	943	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';
Lines 1082-1088 sub _fix_limit_special_cases { Link Here
1082		1091
1083	my @new_lim;	1092	my @new_lim;
1084	foreach my $l (@$limits) {	1093	foreach my $l (@$limits) {
1085
1086	# This is set up by opac-search.pl	1094	# This is set up by opac-search.pl
1087	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {	1095	if ( $l =~ /^yr,st-numeric,ge[=:]/ ) {
1088	my ( $start, $end ) =	1096	my ( $start, $end ) =
Lines 1335-1338 sub _search_fields { Link Here
1335	}	1343	}
1336	}	1344	}
1337		1345
		1346	sub _rebuild_to_es_advanced_query {
		1347	my ($res) = @_;
		1348	my $query_string = $res->{query}->{query_string};
		1349	$query_string->{query} = '*' unless $query_string->{query};
		1350	delete $res->{query}->{query_string};
		1351
		1352	my %filter;
		1353	for my $advanced_query (@$es_advanced_searches) {
		1354	if ( $advanced_query->{field} eq 'geolocation') {
		1355	my ($lat, $lon, $distance) = map { $_ =~ /:(.)\/ } split('\s+', $advanced_query->{operand});
		1356	$filter{geo_distance} = {
		1357	distance => $distance,
		1358	geolocation => {
		1359	lat => $lat,
		1360	lon => $lon,
		1361	}
		1362	};
		1363	}
		1364	else {
		1365	warn "unknown advanced ElasticSearch query: ".join(', ',%$advanced_query);
		1366	}
		1367	}
		1368
		1369	$res->{query} = {
		1370	bool => {
		1371	must => {
		1372	query_string => $query_string
		1373	},
		1374	filter => \%filter,
		1375	}
		1376	};
		1377
		1378	return $res;
		1379	}
		1380
		1381
1338	1;	1382	1;

Lines 91-96 sub search { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (+5 lines)
91	$query->{from} = $page * $query->{size};	91	$query->{from} = $page * $query->{size};
92	}	92	}
93	my $elasticsearch = $self->get_elasticsearch();	93	my $elasticsearch = $self->get_elasticsearch();
		94
		95	# XXX investigate where empty query_string is coming from
		96	delete $query->{query}->{query_string} if
		97	$query->{query}->{query_string} && !%{$query->{query}->{query_string}};
		98
94	my $results = eval {	99	my $results = eval {
95	$elasticsearch->search(	100	$elasticsearch->search(
96	index => $self->index_name,	101	index => $self->index_name,

Lines 41-46 search: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
41	ci_raw:	41	ci_raw:
42	type: keyword	42	type: keyword
43	normalizer: icu_folding_normalizer	43	normalizer: icu_folding_normalizer
		44	geo_point:
		45	type: geo_point
44	default:	46	default:
45	type: text	47	type: text
46	analyzer: analyzer_standard	48	analyzer: analyzer_standard




    opac: 1
    staff_client: 1
    type: ''
  geolocation_lat:
    label: geolocation_lat
    mappings:
      - facet: ''
        marc_field: 034s
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  geolocation_lon:
    label: geolocation_lon
    mappings:
      - facet: ''
        marc_field: 034t
        marc_type: marc21
        sort: 0
        suggestible: ''
    type: geo_point
  holdingbranch:
    facet_order: 8
    label: holdinglibrary




use Modern::Perl;

return {
    bug_number => "31652",
    description => "Add geo-search: new value for search_field.type enum",
    up => sub {
        my ($args) = @_;
        my ($dbh, $out) = @$args{qw(dbh out)};
        # Do you stuffs here
        $dbh->do(q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') });
        # Print useful stuff here
        say $out "Added new value 'geo_point' to search_field.type enum";
    },
};

Lines 5144-5150 CREATE TABLE `search_field` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (-1 / +1 lines)
5144	`id` int(11) NOT NULL AUTO_INCREMENT,	5144	`id` int(11) NOT NULL AUTO_INCREMENT,
5145	`name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',	5145	`name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
5146	`label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',	5146	`label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',
5147	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',	5147	`type` enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
5148	`weight` decimal(5,2) DEFAULT NULL,	5148	`weight` decimal(5,2) DEFAULT NULL,
5149	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',	5149	`facet_order` tinyint(4) DEFAULT NULL COMMENT 'the order place of the field in facet list if faceted',
5150	`staff_client` tinyint(1) NOT NULL DEFAULT 1,	5150	`staff_client` tinyint(1) NOT NULL DEFAULT 1,

Lines 217-222 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
217	[% ELSE %]	217	[% ELSE %]
218	<option value="callnumber">Call Number</option>	218	<option value="callnumber">Call Number</option>
219	[% END %]	219	[% END %]
		220	[% IF search_field.type == "geo_point" %]
		221	<option value="geo_point" selected="selected">Geo Point</option>
		222	[% ELSE %]
		223	<option value="geo_point">Geo Point</option>
		224	[% END %]
220	</select>	225	</select>
221	</td>	226	</td>
222	<td data-order="[% search_field.weight \| html %]">	227	<td data-order="[% search_field.weight \| html %]">

Lines 106-112 my $format = $cgi->param("format") \|\| ''; Link Here

(-)a/opac/opac-search.pl (-4 / +18 lines)
106	if ($format =~ /(rss\|atom\|opensearchdescription)/) {	106	if ($format =~ /(rss\|atom\|opensearchdescription)/) {
107	$template_name = 'opac-opensearch.tt';	107	$template_name = 'opac-opensearch.tt';
108	}	108	}
109	elsif ((@params>=1) \|\| (defined $cgi->param("q") && $cgi->param("q") ne "") \|\| ($cgi->param('multibranchlimit')) \|\| ($cgi->param('limit-yr')) \|\| @searchCategories ) {	109	elsif ((@params>=1) \|\| (defined $cgi->param("q") && $cgi->param("q") ne "") \|\|
		110	($cgi->param('multibranchlimit')) \|\| ($cgi->param('limit-yr')) \|\| @searchCategories
		111	\|\| $cgi->param('lat') ){
110	$template_name = 'opac-results.tt';	112	$template_name = 'opac-results.tt';
111	}	113	}
112	else {	114	else {
Lines 401-408 my @operators = $cgi->multi_param('op'); Link Here
401		403
402	# indexes are query qualifiers, like 'title', 'author', etc. They	404	# indexes are query qualifiers, like 'title', 'author', etc. They
403	# can be single or multiple parameters separated by comma: kw,right-Truncation	405	# can be single or multiple parameters separated by comma: kw,right-Truncation
		406
		407	if ($params->{'lat'} && $params->{'lng'} && $params->{'distance'} ) {
		408	$params->{q} = sprintf("lat:%s lng:%s distance:%s",
		409	$params->{'lat'}, $params->{'lng'}, $params->{'distance'});
		410
		411	$params->{idx} = 'geolocation';
		412	delete $params->{'lat'};
		413	delete $params->{'lng'};
		414	delete $params->{'radius'};
		415	}
		416
		417
		418
404	my @indexes = $cgi->multi_param('idx');	419	my @indexes = $cgi->multi_param('idx');
405	@indexes = map { uri_unescape($_) } @indexes;	420	@indexes = grep { $_ } @indexes;
		421	@indexes = map { uri_unescape($_) } @indexes;
406		422
407	# if a simple index (only one) display the index used in the top search box	423	# if a simple index (only one) display the index used in the top search box
408	if ($indexes[0] && !$indexes[1]) {	424	if ($indexes[0] && !$indexes[1]) {
Lines 456-462 foreach my $limit(@limits) { Link Here
456	}	472	}
457	$template->param(available => $available);	473	$template->param(available => $available);
458		474
459	# append year limits if they exist
460	if ($params->{'limit-yr'}) {	475	if ($params->{'limit-yr'}) {
461	if ($params->{'limit-yr'} =~ /\d{4}/) {	476	if ($params->{'limit-yr'} =~ /\d{4}/) {
462	push @limits, "yr,st-numeric=$params->{'limit-yr'}";	477	push @limits, "yr,st-numeric=$params->{'limit-yr'}";
463	-

Return to bug 31652

Lines 1855-1860 biblios: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (+18 lines)
1855	opac: 1	1855	opac: 1
1856	staff_client: 1	1856	staff_client: 1
1857	type: ''	1857	type: ''
		1858	geolocation_lat:
		1859	label: geolocation_lat
		1860	mappings:
		1861	- facet: ''
		1862	marc_field: 034s
		1863	marc_type: marc21
		1864	sort: 0
		1865	suggestible: ''
		1866	type: geo_point
		1867	geolocation_lon:
		1868	label: geolocation_lon
		1869	mappings:
		1870	- facet: ''
		1871	marc_field: 034t
		1872	marc_type: marc21
		1873	sort: 0
		1874	suggestible: ''
		1875	type: geo_point
1858	holdingbranch:	1876	holdingbranch:
1859	facet_order: 8	1877	facet_order: 8
1860	label: holdinglibrary	1878	label: holdinglibrary

Lines 48-54 the human readable name of the field, for display Link Here

(-)a/Koha/Schema/Result/SearchField.pm (-3 / +4 lines)
48	=head2 type	48	=head2 type
49		49
50	data_type: 'enum'	50	data_type: 'enum'
51	extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber"]}	51	extra: {list => ["","string","date","number","boolean","sum","isbn","stdno","year","callnumber","geo_point"]}
52	is_nullable: 0	52	is_nullable: 0
53		53
54	what type of data this holds, relevant when storing it in the search engine	54	what type of data this holds, relevant when storing it in the search engine
Lines 109-114 __PACKAGE__->add_columns( Link Here
109	"stdno",	109	"stdno",
110	"year",	110	"year",
111	"callnumber",	111	"callnumber",
		112	"geo_point",
112	],	113	],
113	},	114	},
114	is_nullable => 0,	115	is_nullable => 0,
Lines 169-176 __PACKAGE__->has_many( Link Here
169	);	170	);
170		171
171		172
172	# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-07-18 15:10:43	173	# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-09-29 12:18:11
173	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uk5JsfPJo0XVvGfMfJg3cg	174	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xtsGkd3Gjnqw8ZDmoeRDzQ
174		175
175	__PACKAGE__->add_columns(	176	__PACKAGE__->add_columns(
176	'+mandatory' => { is_boolean => 1 },	177	'+mandatory' => { is_boolean => 1 },

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_31652_add_geo_search.pl (+14 lines)
	1	use Modern::Perl;
	2
	3	return {
	4	bug_number => "31652",
	5	description => "Add geo-search: new value for search_field.type enum",
	6	up => sub {
	7	my ($args) = @_;
	8	my ($dbh, $out) = @$args{qw(dbh out)};
	9	# Do you stuffs here
	10	$dbh->do(q{ alter table search_field MODIFY COLUMN type enum('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','geo_point') });
	11	# Print useful stuff here
	12	say $out "Added new value 'geo_point' to search_field.type enum";
	13	},
	14	};