@@ -, +, @@ --- Koha/Schema/Result/SearchField.pm | 6 + Koha/Schema/Result/SearchMarcMap.pm | 6 + Koha/SearchEngine/Elasticsearch/Browse.pm | 186 +++++++++++++++++++++ Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 1 - Koha/SearchEngine/Elasticsearch/Search.pm | 13 +- .../opac-tmpl/bootstrap/en/modules/opac-browse.tt | 89 ++++++++++ koha-tmpl/opac-tmpl/bootstrap/js/browse.js | 172 +++++++++++++++++++ koha-tmpl/opac-tmpl/bootstrap/less/opac.less | 101 +++++++++++ opac/opac-browse.pl | 114 +++++++++++++ t/Koha_SearchEngine_Elasticsearch_Browse.t | 68 ++++++++ 10 files changed, 750 insertions(+), 6 deletions(-) create mode 100644 Koha/SearchEngine/Elasticsearch/Browse.pm create mode 100644 koha-tmpl/opac-tmpl/bootstrap/en/modules/opac-browse.tt create mode 100644 koha-tmpl/opac-tmpl/bootstrap/js/browse.js create mode 100755 opac/opac-browse.pl create mode 100755 t/Koha_SearchEngine_Elasticsearch_Browse.t --- a/Koha/Schema/Result/SearchField.pm +++ a/Koha/Schema/Result/SearchField.pm @@ -119,4 +119,10 @@ __PACKAGE__->has_many( __PACKAGE__->many_to_many("search_marc_maps", "search_marc_to_fields", "search_marc_map"); + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pd5chYwH+KRhI2O8/e7bSg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration 1; --- a/Koha/Schema/Result/SearchMarcMap.pm +++ a/Koha/Schema/Result/SearchMarcMap.pm @@ -127,4 +127,10 @@ __PACKAGE__->has_many( __PACKAGE__->many_to_many("search_fields", "search_marc_to_fields", "search_field"); + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/6VCm/kMfCjtmD4Kx3HGMA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration 1; --- a/Koha/SearchEngine/Elasticsearch/Browse.pm +++ a/Koha/SearchEngine/Elasticsearch/Browse.pm @@ -0,0 +1,186 @@ +package Koha::SearchEngine::Elasticsearch::Browse; + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch + +=head1 SYNOPSIS + + my $browser = + Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); + my $results = $browser->browse( + 'prefi', 'title', + { + results => '500', + fuzziness => 2, + } + ); + foreach my $r (@$results) { + push @hits, $r->{text}; + } + +=head1 DESCRIPTION + +This provides an easy interface to the "browse" functionality. Essentially, +it does a fast prefix search on defined fields. The fields have to be marked +as "suggestible" in the database when indexing takes place. + +=head1 METHODS + +=cut + +use base qw(Koha::SearchEngine::Elasticsearch); +use Modern::Perl; + +use Catmandu::Store::ElasticSearch; + +use Carp; +use Data::Dumper; + +Koha::SearchEngine::Elasticsearch::Browse->mk_accessors(qw( store )); + +=head2 browse + + my $results = $browser->browse($prefix, $field, \%options); + +Does a prefix search for C<$prefix>, looking in C<$field>. Options are: + +=over 4 + +=item count + +The number of results to return. For Koha browse purposes, this should +probably be fairly high. Defaults to 500. + +=item fuzziness + +How much allowing for typos and misspellings is done. If 0, then it must match +exactly. If unspecified, it defaults to '1', which is probably the most useful. +Otherwise, it is a number specifying the Levenshtein edit distance relative to +the string length, according to the following lengths: + +=over 4 + +=item 0..2 + +must match exactly + +=item 3..5 + +C edits allowed + +=item >5 + +C+1 edits allowed + +=back + +In all cases the maximum number of edits allowed is two (an elasticsearch +restriction.) + +=back + +=head3 Returns + +This returns an arrayref of hashrefs. Each hashref contains a "text" element +that contains the field as returned. There may be other fields in that +hashref too, but they're less likely to be important. + +The array will be ordered as returned from Elasticsearch, which seems to be +in order of some form of relevance. + +=cut + +sub browse { + my ($self, $prefix, $field, $options) = @_; + + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + ) + ) unless $self->store; + + my $query = $self->_build_query($prefix, $field, $options); + my $results = $self->store->bag->search(%$query); + return $results->{suggest}{suggestions}[0]{options}; +} + +=head2 _build_query + + my $query = $self->_build_query($prefix, $field, $options); + +Arguments are the same as for L. This will return a query structure +for elasticsearch to use. + +=cut + +sub _build_query { + my ( $self, $prefix, $field, $options ) = @_; + + $options = {} unless $options; + my $f = $options->{fuzziness} // 1; + my $l = length($prefix); + my $fuzzie; + if ( $l <= 2 ) { + $fuzzie = 0; + } + elsif ( $l <= 5 ) { + $fuzzie = $f; + } + else { + $fuzzie = $f + 1; + } + $fuzzie = 2 if $fuzzie > 2; + + my $size = $options->{count} // 500; + my $query = { + # this is an annoying thing, if we set size to 0 it gets rewritten + # to 10. There's a bug somewhere in one of the libraries. + size => 1, + suggest => { + suggestions => { + text => $prefix, + completion => { + field => $field . '__suggestion', + size => $size, + fuzzy => { + fuzziness => $fuzzie, + } + } + } + } + }; + return $query; +} + +1; + +__END__ + +=head1 AUTHOR + +=over 4 + +=item Robin Sheat C<< >> + +=back + +=cut --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -47,7 +47,6 @@ use Modern::Perl; use URI::Escape; use C4::Context; -use Data::Dumper; # TODO remove =head2 build_query --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ a/Koha/SearchEngine/Elasticsearch/Search.pm @@ -78,17 +78,20 @@ Returns =cut sub search { - my ($self, $query, $page, $count, %options) = @_; + my ( $self, $query, $page, $count, %options ) = @_; my $params = $self->get_elasticsearch_params(); my %paging; + # 20 is the default number of results per page $paging{limit} = $count || 20; + # ES/Catmandu doesn't want pages, it wants a record to start from. - if (exists $options{offset}) { + if ( exists $options{offset} ) { $paging{start} = $options{offset}; - } else { - $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1; + } + else { + $page = ( !defined($page) || ( $page <= 0 ) ) ? 0 : $page - 1; $paging{start} = $page * $paging{limit}; } $self->store( @@ -165,7 +168,7 @@ sub search_compat { # consumers of this expect a name-spaced result, we provide the default # configuration. my %result; - $result{biblioserver}{hits} = $results->total; + $result{biblioserver}{hits} = $results->total; $result{biblioserver}{RECORDS} = \@records; return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet)); } --- a/koha-tmpl/opac-tmpl/bootstrap/en/modules/opac-browse.tt +++ a/koha-tmpl/opac-tmpl/bootstrap/en/modules/opac-browse.tt @@ -0,0 +1,89 @@ +[% USE Koha %] +[% INCLUDE 'doc-head-open.inc' %] +[% IF ( LibraryNameTitle ) %][% LibraryNameTitle %][% ELSE %]Koha online[% END %] catalog › Browse our catalog +[% INCLUDE 'doc-head-close.inc' %] +[% BLOCK cssinclude %][% END %] +[% INCLUDE 'bodytag.inc' bodyid='opac-browser' %] +[% INCLUDE 'masthead.inc' %] + + +
+ + +
+
+ [% IF ( OpacNav || OpacNavBottom ) %] + +
+ +
+ [% END %] + + [% IF ( OpacNav ) %] + +
+ [% ELSE %] + +
+ [% END %] + + +
+
+
+
+[% INCLUDE 'opac-bottom.inc' %] +[% BLOCK jsinclude %] + +[% END %] --- a/koha-tmpl/opac-tmpl/bootstrap/js/browse.js +++ a/koha-tmpl/opac-tmpl/bootstrap/js/browse.js @@ -0,0 +1,172 @@ +jQuery.fn.overflowScrollReset = function() { + $(this).scrollTop($(this).scrollTop() - $(this).offset().top); + return this; +}; + +$(document).ready(function(){ + var xhrGetSuggestions, xhrGetResults; + + $('#browse-search form').submit(function(event) { + // if there is an in progress request, abort it so we + // don't end up with a race condition + if(xhrGetSuggestions && xhrGetSuggestions.readyState != 4){ + xhrGetSuggestions.abort(); + } + + var userInput = $('#browse-searchterm').val().trim(); + var userField = $('#browse-searchfield').val(); + var userFuzziness = $('input[name=browse-searchfuzziness]:checked', '#browse-searchfuzziness').val(); + var leftPaneResults = $('#browse-searchresults li').not('.loading, .no-results'); + var rightPaneResults = $('#browse-selectionsearch ol li'); + + event.preventDefault(); + + if(!userInput) { + return; + } + + // remove any error states and show the results area (except right pane) + $('#browse-suggestionserror').addClass('hidden'); + $('#browse-searchresults .no-results').addClass('hidden'); + $('#browse-resultswrapper').removeClass('hidden'); + $('#browse-selection').addClass('hidden').text(""); + $('#browse-selectionsearch').addClass('hidden'); + + // clear any results from left and right panes + leftPaneResults.remove(); + rightPaneResults.remove(); + + // show the spinner in the left pane + $('#browse-searchresults .loading').removeClass('hidden'); + + xhrGetSuggestions = $.get(window.location.pathname, {api: "GetSuggestions", field: userField, prefix: userInput, fuzziness: userFuzziness}) + .always(function() { + // hide spinner + $('#browse-searchresults .loading').addClass('hidden'); + }) + .done(function(data) { + var fragment = document.createDocumentFragment(); + + if (data.length === 0) { + $('#browse-searchresults .no-results').removeClass('hidden'); + + return; + } + + // scroll to top of container again + $("#browse-searchresults").overflowScrollReset(); + + // store the type of search that was performed as an attrib + $('#browse-searchresults').data('field', userField); + + $.each(data, function(index, object) { + // use a document fragment so we don't need to nest the elems + // or append during each iteration (which would be slow) + var elem = document.createElement("li"); + var link = document.createElement("a"); + link.textContent = object.text; + link.setAttribute("href", "#"); + elem.appendChild(link); + fragment.appendChild(elem); + }); + + $('#browse-searchresults').append(fragment.cloneNode(true)); + }) + .fail(function(jqXHR) { + //if 500 or 404 (abort is okay though) + if (jqXHR.statusText !== "abort") { + $('#browse-resultswrapper').addClass('hidden'); + $('#browse-suggestionserror').removeClass('hidden'); + } + }); + }); + + $('#browse-searchresults').on("click", 'a', function(event) { + // if there is an in progress request, abort it so we + // don't end up with a race condition + if(xhrGetResults && xhrGetResults.readyState != 4){ + xhrGetResults.abort(); + } + + var term = $(this).text(); + var field = $('#browse-searchresults').data('field'); + var rightPaneResults = $('#browse-selectionsearch ol li'); + + event.preventDefault(); + + // clear any current selected classes and add a new one + $(this).parent().siblings().children().removeClass('selected'); + $(this).addClass('selected'); + + // copy in the clicked text + $('#browse-selection').removeClass('hidden').text(term); + + // show the right hand pane if it is not shown already + $('#browse-selectionsearch').removeClass('hidden'); + + // hide the no results element + $('#browse-selectionsearch .no-results').addClass('hidden'); + + // clear results + rightPaneResults.remove(); + + // turn the spinner on + $('#browse-selectionsearch .loading').removeClass('hidden'); + + // do the query for the term + xhrGetResults = $.get(window.location.pathname, {api: "GetResults", field: field, term: term}) + .always(function() { + // hide spinner + $('#browse-selectionsearch .loading').addClass('hidden'); + }) + .done(function(data) { + var fragment = document.createDocumentFragment(); + + if (data.length === 0) { + $('#browse-selectionsearch .no-results').removeClass('hidden'); + + return; + } + + // scroll to top of container again + $("#browse-selectionsearch").overflowScrollReset(); + + $.each(data, function(index, object) { + // use a document fragment so we don't need to nest the elems + // or append during each iteration (which would be slow) + var elem = document.createElement("li"); + var title = document.createElement("h4"); + var link = document.createElement("a"); + var author = document.createElement("p"); + var destination = window.location.pathname; + + destination = destination.replace("browse", "detail"); + destination = destination + "?biblionumber=" + object.id; + + author.className = "author"; + + link.setAttribute("href", destination); + link.setAttribute("target", "_blank"); + link.textContent = object.title; + title.appendChild(link); + + author.textContent = object.author; + + elem.appendChild(title); + elem.appendChild(author); + fragment.appendChild(elem); + }); + + $('#browse-selectionsearch ol').append(fragment.cloneNode(true)); + }) + .fail(function(jqXHR) { + //if 500 or 404 (abort is okay though) + if (jqXHR.statusText !== "abort") { + $('#browse-resultswrapper').addClass('hidden'); + $('#browse-suggestionserror').removeClass('hidden'); + } + }); + + }); + +}); --- a/koha-tmpl/opac-tmpl/bootstrap/less/opac.less +++ a/koha-tmpl/opac-tmpl/bootstrap/less/opac.less @@ -312,6 +312,107 @@ td { } } +/*opac browse search*/ +#browse-search { + + form { + label { + display: inline-block; + margin-right:5px; + } + + [type=submit] { + margin-top: 10px; + } + } + + #browse-resultswrapper { + margin-top: 4em; + + @media (min-width: 768px) and (max-width: 984px) { + margin-top: 2em; + } + + @media (max-width: 767px) { + margin-top: 1em; + } + } + #browse-searchresults, #browse-selectionsearch { + border: 1px solid #E3E3E3; + .border-radius-all(4px); + padding: 0; + overflow-y: auto; + max-height: 31em; + margin-bottom: 2em; + } + #browse-searchresults { + max-height: 31em; + list-style: none; + padding: 10px; + + a { + display: block; + margin-bottom: 5px; + + &.selected { + background-color:#EEE; + } + } + + li:last-child a { + margin-bottom: 0; + } + + @media (max-width: 767px) { + max-height: 13em; + } + } + #browse-selection { + margin-top: -40px; + padding-top: 0; + + @media (max-width: 767px) { + margin-top: 0; + } + } + #browse-selectionsearch ol { + list-style: none; + margin: 0; + + li { + padding: 1em; + + &:nth-child(odd) { + background-color: #F4F4F4; + } + } + } + #browse-selectionsearch p.subjects { + font-size: 0.9em; + margin-bottom: 0; + } + #browse-selectionsearch h4 { + margin: 0; + } + .error, .no-results { + background-color: #EEE; + border: 1px solid #E8E8E8; + text-align: left; + padding: 0.5em; + .border-radius-all(3px); + } + .loading { + text-align: center; + + img { + margin:0.5em 0; + position: relative; + left: -5px; + } + } +} +/*end browse search*/ + /* Override Bootstrap alert */ .alert { background: #fffbe5; /* Old browsers */ --- a/opac/opac-browse.pl +++ a/opac/opac-browse.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl + +# This is a CGI script that handles the browse feature. + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use CGI; + +use C4::Auth; +use C4::Context; +use C4::Output; + +use Koha::SearchEngine::Elasticsearch; +use Koha::SearchEngine::Elasticsearch::Browse; +use Koha::SearchEngine::Elasticsearch::QueryBuilder; +use Koha::SearchEngine::Elasticsearch::Search; + +use JSON; +use Unicode::Collate; + +my $query = new CGI; +binmode STDOUT, ':encoding(UTF-8)'; + +# This is the temporary entrance point to the API. Once bug #13799 is settled, +# it should be ported to using that. +my $api = $query->param('api'); + +if ( !$api ) { + my ( $template, $loggedinuser, $cookie ) = get_template_and_user( + { + template_name => "opac-browse.tt", + query => $query, + type => "opac", + authnotrequired => ( C4::Context->preference("OpacPublic") ? 1 : 0 ), + } + ); + $template->param(); + output_html_with_http_headers $query, $cookie, $template->output; + + +} +elsif ( $api eq 'GetSuggestions' ) { + my $fuzzie = $query->param('fuzziness'); + my $prefix = $query->param('prefix'); + my $field = $query->param('field'); + +# Under a persistent environment, we should probably not reinit this every time. + my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); + my $res = $browser->browse( $prefix, $field, { fuzziness => $fuzzie } ); + + my %seen; + my @sorted = + grep { !$seen{$_->{text}}++ } + sort { lc($a->{text}) cmp lc($b->{text}) } @$res; + print header( + -type => 'application/json', + -charset => 'utf-8' + ); + print to_json( \@sorted ); +} +elsif ( $api eq 'GetResults' ) { + my $term = $query->param('term'); + my $field = $query->param('field'); + + my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new( { index => 'biblios' } ); + my $searcher = Koha::SearchEngine::Elasticsearch::Search->new( + { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); + + my $query = { query => { term => { $field.".raw" => $term } } } ; + my $results = $searcher->search( $query, undef, 500 ); + my @output = _filter_for_output( $results->{hits} ); + print header( + -type => 'application/json', + -charset => 'utf-8' + ); + print to_json( \@output ); +} + +# This is a temporary, MARC21-only thing that will grab titles, and authors +# This should probably be done with some templatey gizmo +# in the future. +sub _filter_for_output { + my ($records) = @_; + my @output; + foreach my $rec (@$records) { + my $biblionumber = $rec->{es_id}; + my $biblio = Koha::Biblios->find({ biblionumber=>$biblionumber }); +warn $biblio->title; + push @output, + { + id => $biblionumber, + title => $biblio->title, + author => $biblio->author, + }; + }; + my @sorted = sort { lc($a->{title}) cmp lc($b->{title}) } @output; + return @sorted; +} --- a/t/Koha_SearchEngine_Elasticsearch_Browse.t +++ a/t/Koha_SearchEngine_Elasticsearch_Browse.t @@ -0,0 +1,68 @@ +#!/usr/bin/perl + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Test::More; + +use_ok('Koha::SearchEngine::Elasticsearch::Browse'); + +# testing browse itself not implemented as it'll require a running ES +can_ok('Koha::SearchEngine::Elasticsearch::Browse', + qw/ _build_query browse /); + +subtest "_build_query tests" => sub { + plan tests => 2; + + my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'}); + my $q = $browse->_build_query('foo', 'title'); + is_deeply($q, { size => 1, + suggest => { + suggestions => { + text => 'foo', + completion => { + field => 'title__suggestion', + size => 500, + fuzzy => { + fuzziness => 1, + } + } + } + } + }, 'No fuzziness or size specified'); + + # Note that a fuzziness of 4 will get reduced to 2. + $q = $browse->_build_query('foo', 'title', { fuzziness => 4, count => 400 }); + is_deeply($q, { size => 1, + suggest => { + suggestions => { + text => 'foo', + completion => { + field => 'title__suggestion', + size => 400, + fuzzy => { + fuzziness => 2, + } + } + } + } + }, 'Fuzziness and size specified'); +}; + +done_testing(); --