@@ -, +, @@ --- Koha/Schema/Result/SearchField.pm | 6 + Koha/Schema/Result/SearchMarcMap.pm | 6 + Koha/SearchEngine/Elasticsearch/Browse.pm | 184 ++++++++++++++++++++++++ Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 101 ++++++++++++- Koha/SearchEngine/Elasticsearch/Search.pm | 43 +++++- opac/opac-browse.pl | 136 ++++++++++++++++++ t/Koha_SearchEngine_Elasticsearch_Browse.t | 68 +++++++++ 7 files changed, 534 insertions(+), 10 deletions(-) create mode 100644 Koha/SearchEngine/Elasticsearch/Browse.pm create mode 100755 opac/opac-browse.pl create mode 100755 t/Koha_SearchEngine_Elasticsearch_Browse.t --- a/Koha/Schema/Result/SearchField.pm +++ a/Koha/Schema/Result/SearchField.pm @@ -119,4 +119,10 @@ __PACKAGE__->has_many( __PACKAGE__->many_to_many("search_marc_maps", "search_marc_to_fields", "search_marc_map"); + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pd5chYwH+KRhI2O8/e7bSg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration 1; --- a/Koha/Schema/Result/SearchMarcMap.pm +++ a/Koha/Schema/Result/SearchMarcMap.pm @@ -127,4 +127,10 @@ __PACKAGE__->has_many( __PACKAGE__->many_to_many("search_fields", "search_marc_to_fields", "search_field"); + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/6VCm/kMfCjtmD4Kx3HGMA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration 1; --- a/Koha/SearchEngine/Elasticsearch/Browse.pm +++ a/Koha/SearchEngine/Elasticsearch/Browse.pm @@ -0,0 +1,184 @@ +package Koha::SearchEngine::Elasticsearch::Browse; + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch + +=head1 SYNOPSIS + + my $browser = + Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); + my $results = $browser->browse( + 'prefi', 'title', + { + results => '500', + fuzziness => 2, + } + ); + foreach my $r (@$results) { + push @hits, $r->{text}; + } + +=head1 DESCRIPTION + +This provides an easy interface to the "browse" functionality. Essentially, +it does a fast prefix search on defined fields. The fields have to be marked +as "suggestible" in the database when indexing takes place. + +=head1 METHODS + +=cut + +use base qw(Koha::ElasticSearch); +use Modern::Perl; + +use Catmandu::Store::ElasticSearch; + +use Carp; +use Data::Dumper; + +Koha::SearchEngine::Elasticsearch::Browse->mk_accessors(qw( store )); + +=head2 browse + + my $results = $browser->browse($prefix, $field, \%options); + +Does a prefix search for C<$prefix>, looking in C<$field>. Options are: + +=over 4 + +=item count + +The number of results to return. For Koha browse purposes, this should +probably be fairly high. Defaults to 500. + +=item fuzziness + +How much allowing for typos and misspellings is done. If 0, then it must match +exactly. If unspecified, it defaults to '1', which is probably the most useful. +Otherwise, it is a number specifying the Levenshtein edit distance relative to +the string length, according to the following lengths: + +=over 4 + +=item 0..2 + +must match exactly + +=item 3..5 + +C edits allowed + +=item >5 + +C+1 edits allowed + +=back + +In all cases the maximum number of edits allowed is two (an elasticsearch +restriction.) + +=head3 Returns + +This returns an arrayref of hashrefs. Each hashref contains a "text" element +that contains the field as returned. There may be other fields in that +hashref too, but they're less likely to be important. + +The array will be ordered as returned from Elasticsearch, which seems to be +in order of some form of relevance. + +=cut + +sub browse { + my ($self, $prefix, $field, $options) = @_; + + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + ) + ) unless $self->store; + + my $query = $self->_build_query($prefix, $field, $options); + my $results = $self->store->bag->search(%$query); + return $results->{suggest}{suggestions}[0]{options}; +} + +=head2 _build_query + + my $query = $self->_build_query($prefix, $field, $options); + +Arguments are the same as for L. This will return a query structure +for elasticsearch to use. + +=cut + +sub _build_query { + my ( $self, $prefix, $field, $options ) = @_; + + $options = {} unless $options; + my $f = $options->{fuzziness} // 1; + my $l = length($prefix); + my $fuzzie; + if ( $l <= 2 ) { + $fuzzie = 0; + } + elsif ( $l <= 5 ) { + $fuzzie = $f; + } + else { + $fuzzie = $f + 1; + } + $fuzzie = 2 if $fuzzie > 2; + + my $size = $options->{count} // 500; + my $query = { + # this is an annoying thing, if we set size to 0 it gets rewritten + # to 10. There's a bug somewhere in one of the libraries. + size => 1, + suggest => { + suggestions => { + text => $prefix, + completion => { + field => $field . '__suggestion', + size => $size, + fuzzy => { + fuzziness => $fuzzie, + } + } + } + } + }; + return $query; +} + +1; + +__END__ + +=head1 AUTHOR + +=over 4 + +=item Robin Sheat C<< >> + +=back + +=cut --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -51,7 +51,97 @@ use Data::Dumper; # TODO remove =head2 build_query - my $simple_query = $builder->build_query("hello", %options) + my $query = $builder->build_query($search, %options); + +C<$search> is an arrayref where each element is a hashref containing +a part of the search. + +The hashref must contain a 'term', which is the string, and may contain other +things to adjust how the search is performed: + +=over 4 + +=item field + +the field to search on, as defined in the C table. If +unspecified, all fields will be searched over. + +=item exact + +do an exact, literal, phrase match rather than a tokenised search. This will +also ignore things like truncation and fuzziness. + +=back + +The C<%options> may be: + +=over 4 + +=item and_or + +either "and" or "or", and determines how multiple terms are combined. Default +is "and." + +=back + +=head3 Returns + +a query structure intended to be passed to elasticsearch. + +=head3 Note + +This is pretty limited, it's intended that more options be added as necessary +to support new functionality. + +=head3 Note 2 + +This ultimately ought to be replaced with a query parser driver, but that code +is not documented so it'll take a while to figure out. + +=cut + +sub build_query { + my ( $self, $search, %options ) = @_; + + my @match_parts; + + foreach my $s (@$search) { + my ( $m_type, $f_suffix, $query_name ); + + # This accounts for different naming of things based on our options + if ( $s->{exact} ) { + $m_type = 'term'; + $f_suffix = '.raw'; + $query_name = 'value'; + } + else { + $m_type = 'match'; + $f_suffix = ''; + $query_name = 'query'; + } + my $field = $s->{field} // "_all"; + my $m = { + $m_type => { + "$s->{field}$f_suffix" => { + $query_name => $s->{term}, + }, + }, + }; + push @match_parts, $m; + } + my $query; + if ( $options{and_or} && lc( $options{and_or} ) eq 'or' ) { + $query->{bool}{should} = \@match_parts; + } + else { + $query->{bool}{must} = \@match_parts; + } + return { query => $query }; +} + +=head2 build_string_query + + my $simple_query = $builder->build_string_query("hello", %options) This will build a query that can be issued to elasticsearch from the provided string input. This expects a lucene style search form (see @@ -72,9 +162,12 @@ according to these values. Valid values for C are 'asc' and 'desc'. =back +Note: This is mostly for compatibility, ideally it shouldn't get used +much. + =cut -sub build_query { +sub build_string_query { my ( $self, $query, %options ) = @_; my $stemming = C4::Context->preference("QueryStemming") || 0; @@ -455,7 +548,7 @@ sub build_authorities_query_compat { Converts the zebra-style sort index information into elasticsearch-style. C<@sort_by> is the same as presented to L, and it returns -something that can be sent to L. +something that can be sent to L. =cut @@ -492,7 +585,7 @@ sub _convert_sort_fields { Converts zebra-style search index notation into elasticsearch-style. C<@indexes> is an array of index names, as presented to L, -and it returns something that can be sent to L. +and it returns something that can be sent to L. B: this will pull from the elasticsearch mappings table to figure out types. --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ a/Koha/SearchEngine/Elasticsearch/Search.pm @@ -74,20 +74,38 @@ an offset (i.e. the number of the record to start with), rather than a page. Returns +=over 4 + +=item records + +an array of hashrefs for each record. Each hashref contains C, which has +a MARC::Record instance, and C, which is the biblionumber of this record. + +=item facets + +facet data from Elasticsearch + +=item hits + +the total number of hits for this search + =cut sub search { - my ($self, $query, $page, $count, %options) = @_; + my ( $self, $query, $page, $count, %options ) = @_; my $params = $self->get_elasticsearch_params(); my %paging; + # 20 is the default number of results per page $paging{limit} = $count || 20; + # ES/Catmandu doesn't want pages, it wants a record to start from. - if (exists $options{offset}) { + if ( exists $options{offset} ) { $paging{start} = $options{offset}; - } else { - $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1; + } + else { + $page = ( !defined($page) || ( $page <= 0 ) ) ? 0 : $page - 1; $paging{start} = $page * $paging{limit}; } $self->store( @@ -101,7 +119,20 @@ sub search { if ($@) { die $self->process_error($@); } - return $results; + my @records; + $results->each( + sub { + # The results come in an array for some reason + my $marc_json = @_[0]->{record}; + my $marc = $self->json2marc($marc_json); + push @records, { marc => $marc, id => @_[0]->{_id} }; + } + ); + return { + records => \@records, + facets => $results->{facets}, + hits => $results->total + }; } =head2 count @@ -165,7 +196,7 @@ sub search_compat { # consumers of this expect a name-spaced result, we provide the default # configuration. my %result; - $result{biblioserver}{hits} = $results->total; + $result{biblioserver}{hits} = $results->{hits}; $result{biblioserver}{RECORDS} = \@records; return (undef, \%result, $self->_convert_facets($results->{facets}, $expanded_facet)); } --- a/opac/opac-browse.pl +++ a/opac/opac-browse.pl @@ -0,0 +1,136 @@ +#!/usr/bin/perl + +# This is a CGI script that handles the browse feature. + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use CGI qw( -utf8 :standard ); + +use C4::Auth; +use C4::Context; + +use Koha::ElasticSearch; +use Koha::SearchEngine::Elasticsearch::Browse; +use Koha::SearchEngine::Elasticsearch::QueryBuilder; +use Koha::SearchEngine::Elasticsearch::Search; + +use JSON; +use Unicode::Collate; + +my $query = new CGI; +binmode STDOUT, ':utf8'; + +# This is the temporary entrance point to the API. Once bug #13799 is settled, +# it should be ported to using that. +my $api = $query->param('api'); + +if ( !$api ) { + + # No parameters, so we just render the template. + +} +elsif ( $api eq 'GetSuggestions' ) { + my $fuzzie = $query->param('fuzziness'); + my $prefix = $query->param('prefix'); + my $field = $query->param('field'); + +# Under a persistant environment, we should probably not reinit this every time. + my $browser = + Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); + my $res = $browser->browse( $prefix, $field, { fuzziness => $fuzzie } ); + my $collator = Unicode::Collate->new(); + + # possible optimisation is to cache the collated values and do a straight + # cmp. + my @sorted = sort { $collator->cmp( $a->{text}, $b->{text} ) } @$res; + print header( + -type => 'application/json', + -charset => 'utf-8' + ); + print to_json( \@sorted ); +} +elsif ( $api eq 'GetResults' ) { + my $term = $query->param('term'); + my $field = $query->param('field'); + + my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new( { index => 'biblios' } ); + my $searcher = Koha::SearchEngine::Elasticsearch::Search->new( + { index => $Koha::ElasticSearch::BIBLIOS_INDEX } ); + + my $query = $builder->build_query( + [ { term => $term, field => $field, exact => 1 } ] ); + my $results = $searcher->search( $query, undef, 500 ); + my @output = _filter_for_output( $results->{records} ); + print header( + -type => 'application/json', + -charset => 'utf-8' + ); + print to_json( \@output ); +} + +# This is a temporary, MARC21-only thing that will grab titles, authors, +# and subjects. This should probably be done with some templatey gizmo +# in the future. +sub _filter_for_output { + no warnings 'qw'; # it doesn't like the , in the qw + my ($records) = @_; + + my @author_marc = qw(100,a 110,a 111,a 700,a 710,a 711,a); + my @subjects_marc = + qw(600 610 611 630 648 650 651); + my @output; + foreach my $rec (@$records) { + my $marc = $rec->{marc}; + my $title; + { + # we do an ugly hack to fix up a little bit of bad formatting. + my ($t_a, $t_b, $t_c) = + (( $marc->subfield( 245, 'a' ) // '' ), + ( $marc->subfield( 245, 'b' ) // '' ), + ( $marc->subfield( 245, 'c' ) // '' )); + $t_a .= ' ' unless $t_a =~ / $/; + $t_b .= ' ' unless $t_b =~ / $/; + $title = $t_a . $t_b . $t_c; + } + my @authors; + foreach my $m (@author_marc) { + push @authors, ( $marc->subfield( split( ',', $m ) ) // () ); + } + my @subj_fields; + foreach my $m (@subjects_marc) { + # first collect all the fields + push @subj_fields, $marc->field($m); + } + # Now grab the a and x from each field and reformat it + my @subjects; + foreach my $s (@subj_fields) { + my @vals = (scalar($s->subfield('a')), $s->subfield('x')); + next unless @vals; + push @subjects, join( ' -- ', @vals ); + } + push @output, + { + id => $rec->{id}, + title => $title, + authors => \@authors, + subjects => \@subjects, + }; + } + return @output; +} --- a/t/Koha_SearchEngine_Elasticsearch_Browse.t +++ a/t/Koha_SearchEngine_Elasticsearch_Browse.t @@ -0,0 +1,68 @@ +#!/usr/bin/perl + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Test::More; + +use_ok('Koha::SearchEngine::Elasticsearch::Browse'); + +# testing browse itself not implemented as it'll require a running ES +can_ok('Koha::SearchEngine::Elasticsearch::Browse', + qw/ _build_query browse /); + +subtest "_build_query tests" => sub { + plan tests => 2; + + my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'}); + my $q = $browse->_build_query('foo', 'title'); + is_deeply($q, { size => 1, + suggest => { + suggestions => { + text => 'foo', + completion => { + field => 'title__suggestion', + size => 500, + fuzzy => { + fuzziness => 1, + } + } + } + } + }, 'No fuzziness or size specified'); + + # Note that a fuzziness of 4 will get reduced to 2. + $q = $browse->_build_query('foo', 'title', { fuzziness => 4, count => 400 }); + is_deeply($q, { size => 1, + suggest => { + suggestions => { + text => 'foo', + completion => { + field => 'title__suggestion', + size => 400, + fuzzy => { + fuzziness => 2, + } + } + } + } + }, 'Fuzziness and size specified'); +}; + +done_testing(); --