From 2c1b51551a363de8d9b7806b0282be51a9eb56a4 Mon Sep 17 00:00:00 2001 From: Ivan Dziuba Date: Thu, 22 Apr 2021 17:30:41 -0400 Subject: [PATCH] Bug 27113: ElasticSearch: Autocomplete in input search Usually the user knows only part of the title of the book or only the name of the author, etc. When he start search something Koha (ElasticSearch) predicts the rest of a word or expression which user is typing. Autocomplete predicts that thanks to index of ElasticSearch. TEST PLAN Important! In this patch we need to do reindex ElasticSearch. ElasticSearch must have all information in his index. 1. Go Intranet -> Preference -> SearchEngine -> ElasticSearch !! APPLY PATCH !! 2. Mapping is good (Intranet -> Catalog -> Search engine configuration (Elasticsearch) ). !Recommended 'Reset Mapping' -> 'Yes' 3. In your koha-conf.xml file you must have good for and version ES 4. Update Preference: ./installer/data/mysql/updatedatabase.pl If that passe good you can look the lines: - DEV atomic update: bug_27113-elasticsearch_autocomplete_input_search.perl - Upgrade to XXX done : Bug 27113 - Autocomplete input on main page with elasticsearch 5. After that we can look two options in the preferences: - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; 4. For add information in the index we must run script for reindexing: ./misc/search_tools/rebuild_elasticsearch.pl -v -d 5. Waiting for the end of indexing 6. Go on Preference and find : - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; Value "Show" turn on autocomplete. 7. Now we have Autocomplete for Intranet/OPAC search input (advanced search also). Signed-off-by: David Nind --- Koha/SearchEngine/Elasticsearch/Browse.pm | 127 +++++++- .../elasticsearch/field_config.yaml | 4 + .../elasticsearch/index_config.yaml | 14 + api/elasticsearch/elasticsearch.pl | 58 ++++ ...asticsearch_autocomplete_input_search.perl | 10 + installer/data/mysql/mandatory/sysprefs.sql | 2 + .../js/elasticsearch/autocomplete.js | 270 ++++++++++++++++++ .../prog/css/elasticsearch/autocomplete.css | 28 ++ .../prog/en/includes/doc-head-close.inc | 5 + .../prog/en/includes/js_includes.inc | 4 + .../modules/admin/preferences/searching.pref | 17 ++ .../prog/en/modules/catalogue/advsearch.tt | 4 + .../intranet-tmpl/prog/js/staff-global.js | 7 + .../opac-elasticsearch/opac-autocomplete.css | 29 ++ .../bootstrap/en/includes/doc-head-close.inc | 5 + .../bootstrap/en/includes/opac-bottom.inc | 4 + .../bootstrap/en/modules/opac-advsearch.tt | 8 + .../opac-elasticsearch/opac-autocomplete.js | 266 +++++++++++++++++ opac/svc/elasticsearch/opac-elasticsearch.pl | 145 ++++++++++ t/Koha_SearchEngine_Elasticsearch_Browse.t | 30 ++ 20 files changed, 1036 insertions(+), 1 deletion(-) create mode 100755 api/elasticsearch/elasticsearch.pl create mode 100644 installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl create mode 100644 koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js create mode 100644 koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css create mode 100644 koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css create mode 100644 koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js create mode 100755 opac/svc/elasticsearch/opac-elasticsearch.pl diff --git a/Koha/SearchEngine/Elasticsearch/Browse.pm b/Koha/SearchEngine/Elasticsearch/Browse.pm index 85070730a7..78f93832c7 100644 --- a/Koha/SearchEngine/Elasticsearch/Browse.pm +++ b/Koha/SearchEngine/Elasticsearch/Browse.pm @@ -40,7 +40,7 @@ Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch This provides an easy interface to the "browse" functionality. Essentially, it does a fast prefix search on defined fields. The fields have to be marked -as "suggestible" in the database when indexing takes place. +as "suggestible" in the database when indexing takes place(no action required for autocomplete). =head1 METHODS @@ -162,6 +162,129 @@ sub _build_query { return $query; } +=head2 autocomplete_one_idx + + my $query = $self->autocomplete_one_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (only one prefix), looking for C<$cgi_q> , using analyzer C<$analyzer> , +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref of hashrefs with highlights. Each hashref contains a "text" element that contains the field as returned. + +=cut + +sub autocomplete_one_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my @source; + my $elasticsearch = $self->get_elasticsearch(); + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + my $res = $elasticsearch->search( + index => $self->index_name, + body => $query + ); + $res->{'val'} = $cgi_q; + $res->{'prefix'} = $prefix; + $res->{'token_counter'} = $token_counter; + + return $res; +} + +=head2 autocomplete_idx + + my $query = $self->autocomplete_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (many prefix), looking for C<$cgi_q>, using analyzer C<$analyzer>, +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub autocomplete_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my %results; + my $idx = 0; + foreach my $pref ( @$prefix ) { + $results{$idx} = $self->autocomplete_one_idx($cgi_q, $pref, $analyzer, $token_counter); + $idx++; + } + $results{'val'} = $cgi_q; + $results{'prefix'} = join( ',', @$prefix ); + $results{'token_counter'} = $token_counter; + return \%results; +} + +=head2 _build_query_autocomplete + + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + +Arguments: + +=over 4 + +=item cgi_q + +GET request + +=item prefix + +Field(s) for autocomplete (title, author, etc...) + +=item analyzer + +Name of analyzer wich we use for autocomplete + +=back + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub _build_query_autocomplete { + my ($self, $cgi_q, $prefix, $analyzer) = @_; + my (@source); + #prefix + analyzer + my $prefix_analyzer = $prefix . '.' . $analyzer; + # we can change these variables + my ($nb_fragments, $size_fragment, $pre_tags, $post_tags) = (1, 100, [""], [""]); + push(@source, $prefix); + my $query = { + _source => \@source, + query => { + match => { + $prefix_analyzer => { + query => $cgi_q, + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => $nb_fragments, + fragment_size => $size_fragment, + pre_tags => $pre_tags, + post_tags => $post_tags, + fields => { + $prefix_analyzer => {} + } + } + }; + return $query; +} + 1; __END__ @@ -172,6 +295,8 @@ __END__ =item Robin Sheat << >> +=item Ivan Dziuba << >> + =back =cut diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index d78bdf06dd..cae35c9585 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -47,6 +47,10 @@ search: type: text analyzer: analyzer_phrase search_analyzer: analyzer_phrase + autocomplete: + type: text + analyzer: autocomplete + search_analyzer: standard raw: type: keyword normalizer: nfkc_cf_normalizer diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index b8b2ac6071..ef6eaa5f48 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -2,6 +2,14 @@ # Index configuration that defines how different analyzers work. index: analysis: + tokenizer: + autocomplete_tokenizer: + type: edge_ngram + min_gram: 1 + max_gram: 16 + token_chars: + - letter + - digit analyzer: # Phrase analyzer is used for phrases (exact phrase match) analyzer_phrase: @@ -10,6 +18,12 @@ index: - icu_folding char_filter: - punctuation + autocomplete: + type: custom + filter: + - icu_folding + - lowercase + tokenizer: autocomplete_tokenizer analyzer_standard: tokenizer: icu_tokenizer filter: diff --git a/api/elasticsearch/elasticsearch.pl b/api/elasticsearch/elasticsearch.pl new file mode 100755 index 0000000000..655bdb568f --- /dev/null +++ b/api/elasticsearch/elasticsearch.pl @@ -0,0 +1,58 @@ +#!/usr/bin/perl + +use Modern::Perl; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +if ($session->param("key") eq "autocomplete") { + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + if ($length >= 1) { + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + print $cgi->header("application/json;charset=UTF-8"); + print to_json($res, {utf8 => 1}); + } + #no prefix 404 + else { + response404JSON(); + } +} else { + response404JSON(); +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json;charset=UTF-8"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} diff --git a/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl b/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl new file mode 100644 index 0000000000..fafe0894d3 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl @@ -0,0 +1,10 @@ +$DBversion = 'XXX'; # will be replaced by the RM +if( CheckVersion( $DBversion ) ) { + # you can use $dbh here like: + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('OPACAutocompleteElasticSearch', '0', NULL, 'If ON, show search suggestions in the OPAC when using Elasticsearch.', 'YesNo')}); + + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('IntranetAutocompleteElasticSearch', '0', NULL, 'If ON, show search suggestions in the staff interface when using Elasticsearch.', 'YesNo')}); + + # Always end with this (adjust the bug info) + NewVersion( $DBversion, 27113, "Autocomplete with Elasticsearch"); +} diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index 04d36ff9e1..004809b90d 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -286,6 +286,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('OnSiteCheckouts','0','','Enable/Disable the on-site checkouts feature','YesNo'), ('OnSiteCheckoutsForce','0','','Enable/Disable the on-site for all cases (Even if a user is debarred, etc.)','YesNo'), ('OnSiteCheckoutAutoCheck','0','','Enable/Do not enable onsite checkout by default if last checkout was an onsite checkout','YesNo'), +('IntranetAutocompleteElasticSearch','0',NULL,'If ON, show search suggestions in the staff interface when using Elasticsearch.','YesNo'), ('IntranetBiblioDefaultView','normal','normal|marc|isbd|labeled_marc','Choose the default detail view in the staff interface; choose between normal, labeled_marc, marc or isbd','Choice'), ('intranetbookbag','1','','If ON, enables display of Cart feature in the intranet','YesNo'), ('IntranetCirculationHomeHTML', '', NULL, 'Show the following HTML in a div on the bottom of the reports home page', 'Free'), @@ -407,6 +408,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('OPACAllowUserToChooseBranch','1','1','Allow the user to choose the branch they want to pickup their hold from','YesNo'), ('OPACAmazonCoverImages','0','','Display cover images on OPAC from Amazon Web Services','YesNo'), ('OpacAuthorities','1',NULL,'If ON, enables the search authorities link on OPAC','YesNo'), +('OPACAutocompleteElasticSearch','0',NULL,'If ON, show search suggestions in the OPAC when using Elasticsearch.','YesNo'), ('OPACBaseURL','',NULL,'Specify the Base URL of the OPAC, e.g., http://opac.mylibrary.com, including the protocol (http:// or https://). Otherwise, the http:// will be added automatically by Koha upon saving.','Free'), ('opacbookbag','1','','If ON, enables display of Cart feature','YesNo'), ('OpacBrowser','0',NULL,'If ON, enables subject authorities browser on OPAC (needs to set misc/cronjob/sbuild_browser_and_cloud.pl)','YesNo'), diff --git a/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js b/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js new file mode 100644 index 0000000000..6c9ff58330 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js @@ -0,0 +1,270 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +/* stop class for elements name=["q"] */ +var stop_class_input = ["form-field-value"]; + +var url_request = '/cgi-bin/koha/api/elasticsearch/elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitIntranet(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + if (!stop_class_input.includes(input_q[nb].className)){ + autocomplete(input_q[nb], nb, left, right, top); + } + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + + var timer; + var doneTimerInterval = 500; + + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + clearTimeout(timer); + timer = setTimeout( function() { + var a, val = inp.value; + /* var for async compare */ + var tmp_input = inp.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", inp.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + inp.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + contentType: "application/json;charset=UTF-8", + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + + myset = GetSetAutocomplete(data, prefix, key); + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(inp); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }, doneTimerInterval); + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for all prefix */ + function GetSetAutocomplete(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#advanced-search input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitIntranet(); diff --git a/koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css b/koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css new file mode 100644 index 0000000000..40dfefb0d6 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css @@ -0,0 +1,28 @@ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + display: inline-block; +} +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + /*position the autocomplete items to be the same width as the container:*/ + } + .autocomplete-items div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + } + .autocomplete-items div:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + .autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: #cedfb1 !important; + color: #ffffff; + } \ No newline at end of file diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc index b35c001820..78a497eab4 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc @@ -26,6 +26,11 @@ [% Asset.css("css/print.css", { media = "print" }) | $raw %] [% INCLUDE intranetstylesheet.inc %] [% IF ( bidi ) %][% Asset.css("css/right-to-left.css") | $raw %][% END %] + +[% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] + [% SET Optylesheet = 'elasticsearch/autocomplete.css' %] + +[% END %] [% END %] + +[% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] +[% Asset.js("js/elasticsearch/autocomplete.js") | $raw %] +[% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index 69619fd6f8..2e9d0a6d3e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -298,3 +298,20 @@ Searching: - LIBRIS base URL - pref: LibrisURL - "Please only change this if you are sure it needs changing." + - + - pref: OPACAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for OPAC. + - + - pref: IntranetAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for Intranet. + - diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt index 29808258d1..7f7dc96725 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt @@ -382,6 +382,10 @@ var dad = line.parentNode; dad.appendChild(line.cloneNode(true)); line.removeChild(ButtonPlus); + /* Intranet JS IntranetAutocompleteElasticSearch */ + [% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] + AutocompleteInitIntranet(); + [% END %] } var Sticky; diff --git a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js b/koha-tmpl/intranet-tmpl/prog/js/staff-global.js index f2becd2a7f..1fd4987e4e 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js +++ b/koha-tmpl/intranet-tmpl/prog/js/staff-global.js @@ -89,6 +89,13 @@ $(document).ready(function() { $(".keep_text").on("click",function(){ var field_index = $(this).parent().index(); keep_text( field_index ); + /* IntranetAutocompleteElasticSearch Tab */ + var tab = this.hash.substr(1, this.hash.length-1); + /* Koha.Preference('IntranetAutocompleteElasticSearch') == Show */ + if (typeof AutocompleteInitIntranet !== "undefined" && tab === 'catalog_search' ){ + AutocompleteInitIntranet(); + } + $("#search-form").focus(); }); $(".toggle_element").on("click",function(e){ diff --git a/koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css b/koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css new file mode 100644 index 0000000000..2b8a566568 --- /dev/null +++ b/koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css @@ -0,0 +1,29 @@ +/* CSS file OPACAutocompleteElasticSearch */ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + display: inline-block; + width: 100%; +} +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + } + .autocomplete-items div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + } + .autocomplete-items div:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + .autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: #cedfb1 !important; + color: #ffffff; + } \ No newline at end of file diff --git a/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc b/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc index e29568d76f..b6b363fba6 100644 --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc +++ b/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc @@ -23,6 +23,11 @@ [% SET opaclayoutstylesheet = 'opac.css' %] [% END %] [% END %] + +[% IF ( Koha.Preference('OPACAutocompleteElasticSearch') ) %] + [% SET Optylesheet = 'opac-elasticsearch/opac-autocomplete.css' %] + +[% END %] [% IF (opaclayoutstylesheet.match('^https?:|^\/')) %] [% ELSE %] diff --git a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc index 4fab59077d..e59582fbf2 100644 --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc +++ b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc @@ -289,6 +289,10 @@ $(document).ready(function() { }); [% PROCESS jsinclude %] + +[% IF ( Koha.Preference('OPACAutocompleteElasticSearch') ) %] + [% Asset.js("js/opac-elasticsearch/opac-autocomplete.js") | $raw %] +[% END %] [% IF ( Koha.Preference('OPACUserJS') ) %] diff --git a/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js b/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js new file mode 100644 index 0000000000..065a76017b --- /dev/null +++ b/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js @@ -0,0 +1,266 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +var url_request = '/cgi-bin/koha/svc/elasticsearch/opac-elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitOpac(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + autocomplete(input_q[nb], nb, left, right, top); + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + $(div_relative).css("display", "inline-block"); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + + var timer; + var doneTimerInterval = 500; + + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + clearTimeout(timer); + timer = setTimeout( function() { + var a, val = inp.value; + /* var for async compare */ + var tmp_input = inp.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", inp.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + inp.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + contentType: "application/json;charset=UTF-8", + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + + myset = GetSetAutocomplete(data, prefix, key); + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(inp); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }, doneTimerInterval); + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for all prefix */ + function GetSetAutocomplete(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#booleansearch input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitOpac(); diff --git a/opac/svc/elasticsearch/opac-elasticsearch.pl b/opac/svc/elasticsearch/opac-elasticsearch.pl new file mode 100755 index 0000000000..1ab233d284 --- /dev/null +++ b/opac/svc/elasticsearch/opac-elasticsearch.pl @@ -0,0 +1,145 @@ +#!/usr/bin/perl + +use Modern::Perl; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +use Koha::Items; +use C4::Context; +use C4::Biblio qw ( GetMarcBiblio ); + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +if ($session->param("key") eq "autocomplete") { + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + if ($length >= 1) { + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + + filterAutocomplete($res); + + print $cgi->header("application/json;charset=UTF-8"); + print to_json($res, {utf8 => 1}); + } + #no prefix 404 + else { + response404JSON(); + } +} else { + response404JSON(); +} + +sub filterAutocomplete { + if (C4::Context->preference('OpacSuppression') || C4::Context->yaml_preference('OpacHiddenItems')) { + my $res = shift; + my @prefix = $res->{ "prefix" }; + @prefix = split(',', $prefix[0]); + + for (my $i = 0; $i < scalar @prefix; $i++) { + my $hits = $res->{ $i }->{ 'hits' }; + my $hitlist = $hits->{ "hits" }; + if (@{$hitlist}) { + # Remove item inside hits in elasticsearch response if the item has + # marc field 942$n set to true and OpacSuppression preference on + if (C4::Context->preference('OpacSuppression')) { + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $record = GetMarcBiblio({ + biblionumber => $hitlist->[$i]->{ "_id" }, + opac => 1 + }); + my $opacsuppressionfield = '942'; + my $opacsuppressionfieldvalue = $record->field($opacsuppressionfield); + if ( $opacsuppressionfieldvalue && + $opacsuppressionfieldvalue->subfield("n") && + $opacsuppressionfieldvalue->subfield("n") == 1) { + # if OPAC suppression by IP address + if (C4::Context->preference('OpacSuppressionByIPRange')) { + my $IPAddress = $ENV{'REMOTE_ADDR'}; + my $IPRange = C4::Context->preference('OpacSuppressionByIPRange'); + if ($IPAddress !~ /^$IPRange/) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } else { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + } + # Remove item inside hits in elasticsearch response if the item is + # declared hidden in OPACHiddenItems preference + if (C4::Context->yaml_preference('OpacHiddenItems')) { + my @biblionumbers; + foreach (@{$hitlist}) { + push(@biblionumbers, $_->{ "_id" }); + } + my $autocomplete_items = Koha::Items->search({ + biblionumber => { -in => \@biblionumbers } + }); + my $filtered_items = $autocomplete_items->filter_by_visible_in_opac({ + patron => undef + }); + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $item = $filtered_items->find({ + biblionumber => $hitlist->[$i]->{ "_id" } + }); + if (!$item) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + # Adjust the max_score inside hits in elasticsearch response + my $maxscore = 0; + foreach ( @{$hitlist} ) { + my $score = $_->{"_score"}; + $maxscore = $score if ($maxscore < $score); + } + if ($maxscore == 0) { + $hits->{ "max_score" } = undef; + } else { + $hits->{ "max_score" } = $maxscore; + } + } + } + } +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json;charset=UTF-8"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} diff --git a/t/Koha_SearchEngine_Elasticsearch_Browse.t b/t/Koha_SearchEngine_Elasticsearch_Browse.t index 986c4ba658..4e93e2c42d 100755 --- a/t/Koha_SearchEngine_Elasticsearch_Browse.t +++ b/t/Koha_SearchEngine_Elasticsearch_Browse.t @@ -65,4 +65,34 @@ subtest "_build_query tests" => sub { }, 'Fuzziness and size specified'); }; +subtest "_build_query_autocomplete tests" => sub { + plan tests => 1; + + my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'}); + + my $q = $browse->_build_query_autocomplete('a', 'title', 'autocomplete'); + + is_deeply($q, { + _source => ["title"], + query => { + match => { + "title.autocomplete" => { + query => 'a', + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => 1, + fragment_size => 100, + pre_tags => [""], + post_tags => [""], + fields => { + "title.autocomplete" => {} + } + } + }, 'Autocomplete for title is specified'); + +}; + done_testing(); -- 2.25.1