From aa436bd689d3015542f61e620349ec962ba158dd Mon Sep 17 00:00:00 2001 From: Ivan Dziuba Date: Thu, 22 Apr 2021 17:30:41 -0400 Subject: [PATCH] Bug 27113: ElasticSearch: Autocomplete in input search Usually the user knows only part of the title of the book or only the name of the author, etc. When he start search something Koha (ElasticSearch) predicts the rest of a word or expression which user is typing. Autocomplete predicts that thanks to index of ElasticSearch. TEST PLAN Important! In this patch we need to do reindex ElasticSearch. ElasticSearch must have all information in his index. 1. Go Intranet -> Preference -> SearchEngine -> ElasticSearch !! APPLY PATCH !! 2. Mapping is good (Intranet -> Catalog -> Search engine configuration (Elasticsearch) ). !Recommended 'Reset Mapping' -> 'Yes' 3. In your koha-conf.xml file you must have good for and version ES 4. Update Preference: ./installer/data/mysql/updatedatabase.pl If that passe good you can look the lines: - DEV atomic update: bug_27113-elasticsearch_autocomplete_input_search.perl - Upgrade to XXX done : Bug 27113 - Autocomplete input on main page with elasticsearch 5. After that we can look two options in the preferences: - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; 4. For add information in the index we must run script for reindexing: ./misc/search_tools/rebuild_elasticsearch.pl -v -d 5. Waiting for the end of indexing 6. Go on Preference and find : - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; Value "Show" turn on autocomplete. 7. Now we have Autocomplete for Intranet/OPAC search input (advanced search also). Signed-off-by: David Nind Bug 27113: (follow-up) Move new CSS to main SCSS files This patch removes the separate CSS files added for the autocomplete feature and puts them into the "main" SCSS files. I think a separate file isn't necessary because the amount of CSS it adds is so small. I've also tweaked the style of the autocomplete menu shown when you use the arrow keys to navigate through the autocomplete choices. I think the previous white-on-light-green didn't have enough contrast. To test you must rebuild the OPAC and staff client CSS: https://wiki.koha-community.org/wiki/Working_with_SCSS_in_the_OPAC_and_staff_client Follow the previous test plan, being careful to observe how it works when the autocomplete menu has been triggered and you use the arrow keys to navigate through the results. Signed-off-by: David Nind --- Koha/SearchEngine/Elasticsearch/Browse.pm | 127 +++++++- .../elasticsearch/field_config.yaml | 4 + .../elasticsearch/index_config.yaml | 14 + api/elasticsearch/elasticsearch.pl | 67 +++++ ...asticsearch_autocomplete_input_search.perl | 10 + installer/data/mysql/mandatory/sysprefs.sql | 4 +- .../js/elasticsearch/autocomplete.js | 283 ++++++++++++++++++ .../prog/css/src/staff-global.scss | 33 ++ .../prog/en/includes/js_includes.inc | 4 + .../modules/admin/preferences/searching.pref | 17 ++ .../prog/en/modules/catalogue/advsearch.tt | 4 + .../intranet-tmpl/prog/js/staff-global.js | 7 + .../opac-tmpl/bootstrap/css/src/opac.scss | 34 +++ .../bootstrap/en/includes/opac-bottom.inc | 4 + .../bootstrap/en/modules/opac-advsearch.tt | 8 + .../opac-elasticsearch/opac-autocomplete.js | 279 +++++++++++++++++ opac/svc/elasticsearch/opac-elasticsearch.pl | 162 ++++++++++ t/Koha_SearchEngine_Elasticsearch_Browse.t | 30 ++ 18 files changed, 1089 insertions(+), 2 deletions(-) create mode 100755 api/elasticsearch/elasticsearch.pl create mode 100644 installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl create mode 100644 koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js create mode 100644 koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js create mode 100755 opac/svc/elasticsearch/opac-elasticsearch.pl diff --git a/Koha/SearchEngine/Elasticsearch/Browse.pm b/Koha/SearchEngine/Elasticsearch/Browse.pm index 85070730a7..78f93832c7 100644 --- a/Koha/SearchEngine/Elasticsearch/Browse.pm +++ b/Koha/SearchEngine/Elasticsearch/Browse.pm @@ -40,7 +40,7 @@ Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch This provides an easy interface to the "browse" functionality. Essentially, it does a fast prefix search on defined fields. The fields have to be marked -as "suggestible" in the database when indexing takes place. +as "suggestible" in the database when indexing takes place(no action required for autocomplete). =head1 METHODS @@ -162,6 +162,129 @@ sub _build_query { return $query; } +=head2 autocomplete_one_idx + + my $query = $self->autocomplete_one_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (only one prefix), looking for C<$cgi_q> , using analyzer C<$analyzer> , +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref of hashrefs with highlights. Each hashref contains a "text" element that contains the field as returned. + +=cut + +sub autocomplete_one_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my @source; + my $elasticsearch = $self->get_elasticsearch(); + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + my $res = $elasticsearch->search( + index => $self->index_name, + body => $query + ); + $res->{'val'} = $cgi_q; + $res->{'prefix'} = $prefix; + $res->{'token_counter'} = $token_counter; + + return $res; +} + +=head2 autocomplete_idx + + my $query = $self->autocomplete_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (many prefix), looking for C<$cgi_q>, using analyzer C<$analyzer>, +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub autocomplete_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my %results; + my $idx = 0; + foreach my $pref ( @$prefix ) { + $results{$idx} = $self->autocomplete_one_idx($cgi_q, $pref, $analyzer, $token_counter); + $idx++; + } + $results{'val'} = $cgi_q; + $results{'prefix'} = join( ',', @$prefix ); + $results{'token_counter'} = $token_counter; + return \%results; +} + +=head2 _build_query_autocomplete + + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + +Arguments: + +=over 4 + +=item cgi_q + +GET request + +=item prefix + +Field(s) for autocomplete (title, author, etc...) + +=item analyzer + +Name of analyzer wich we use for autocomplete + +=back + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub _build_query_autocomplete { + my ($self, $cgi_q, $prefix, $analyzer) = @_; + my (@source); + #prefix + analyzer + my $prefix_analyzer = $prefix . '.' . $analyzer; + # we can change these variables + my ($nb_fragments, $size_fragment, $pre_tags, $post_tags) = (1, 100, [""], [""]); + push(@source, $prefix); + my $query = { + _source => \@source, + query => { + match => { + $prefix_analyzer => { + query => $cgi_q, + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => $nb_fragments, + fragment_size => $size_fragment, + pre_tags => $pre_tags, + post_tags => $post_tags, + fields => { + $prefix_analyzer => {} + } + } + }; + return $query; +} + 1; __END__ @@ -172,6 +295,8 @@ __END__ =item Robin Sheat << >> +=item Ivan Dziuba << >> + =back =cut diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index d78bdf06dd..cae35c9585 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -47,6 +47,10 @@ search: type: text analyzer: analyzer_phrase search_analyzer: analyzer_phrase + autocomplete: + type: text + analyzer: autocomplete + search_analyzer: standard raw: type: keyword normalizer: nfkc_cf_normalizer diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index b8b2ac6071..ab1e5a631b 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -2,6 +2,14 @@ # Index configuration that defines how different analyzers work. index: analysis: + tokenizer: + autocomplete_tokenizer: + type: edge_ngram + min_gram: 1 + max_gram: 10 + token_chars: + - letter + - digit analyzer: # Phrase analyzer is used for phrases (exact phrase match) analyzer_phrase: @@ -10,6 +18,12 @@ index: - icu_folding char_filter: - punctuation + autocomplete: + type: custom + filter: + - icu_folding + - lowercase + tokenizer: autocomplete_tokenizer analyzer_standard: tokenizer: icu_tokenizer filter: diff --git a/api/elasticsearch/elasticsearch.pl b/api/elasticsearch/elasticsearch.pl new file mode 100755 index 0000000000..718af730f4 --- /dev/null +++ b/api/elasticsearch/elasticsearch.pl @@ -0,0 +1,67 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +if ($session->param("key") eq "autocomplete") { + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + #search by many prefix fields + if ($length > 1){ + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + print $cgi->header("application/json"); + print to_json($res); + } + #search by one prefix field + elsif ($length == 1) { + my $res = $browser->autocomplete_one_idx($ses, $prefix[0], $session->param("analyzer"), $session->param("token_counter")); + print $cgi->header("application/json"); + print to_json($res); + } + #no prefix 404 + else { + response404JSON(); + } +} +else { + response404JSON(); +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} diff --git a/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl b/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl new file mode 100644 index 0000000000..c5947f6f74 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl @@ -0,0 +1,10 @@ +$DBversion = 'XXX'; # will be replaced by the RM +if( CheckVersion( $DBversion ) ) { + # you can use $dbh here like: + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('OPACAutocompleteElasticSearch', '0', NULL, NULL, 'YesNo')}); + + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('IntranetAutocompleteElasticSearch', '0', NULL, NULL, 'YesNo')}); + + # Always end with this (adjust the bug info) + NewVersion( $DBversion, 27113, "Autocomplete with elasticsearch"); +} diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index a15eb6c112..a6548aa2a7 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -754,5 +754,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('XSLTListsDisplay','default','','Enable XSLT stylesheet control over lists pages display on intranet','Free'), ('XSLTResultsDisplay','default','','Enable XSL stylesheet control over results page display on intranet','Free'), ('z3950AuthorAuthFields','701,702,700',NULL,'Define the MARC biblio fields for Personal Name Authorities to fill biblio.author','free'), -('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo') +('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo'), +('OPACAutocompleteElasticSearch','0',NULL,NULL,'YesNo'), +('IntranetAutocompleteElasticSearch','0',NULL,NULL,'YesNo') ; diff --git a/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js b/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js new file mode 100644 index 0000000000..a6fb4df66b --- /dev/null +++ b/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js @@ -0,0 +1,283 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +/* stop class for elements name=["q"] */ +var stop_class_input = ["form-field-value"]; + +var url_request = '/cgi-bin/koha/api/elasticsearch/elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitIntranet(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + if (!stop_class_input.includes(input_q[nb].className)){ + autocomplete(input_q[nb], nb, left, right, top); + } + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + var a, val = this.value; + /* var for async compare */ + var tmp_input = this.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", this.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + this.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + /* autocomplete for all prefix */ + if (chose_prefix === 'kw' || chose_prefix === ''){ + myset = GetSetAutocompleteAllIdx(data, prefix, key); + } else { // autocomplete for one prefix + myset = GetSetAutocompleteOneIdx(data, prefix, key); + }; + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(this); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for only one prefix title/author/etc... */ + function GetSetAutocompleteOneIdx(data, prefix, key){ + let myset = new Set(); + let tmp_data = data['hits']['hits']; + for (let i = 0; i < tmp_data.length; i++) { + for (let j = 0; j < tmp_data[i]['highlight'][prefix + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][prefix + '.' + key][j]; + tmp = tmp.replace(/^\[/g, ''); + tmp = tmp.replace(/\]+$/g, ''); + myset.add(tmp.replace(/^[ &\/\\#,+)$~%.'":*?>{}!;]+|[ &\/\\#,+($~%.'":*?<{}!;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + return myset; + }; + /* get autocomplete for all prefix */ + function GetSetAutocompleteAllIdx(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#advanced-search input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitIntranet(); diff --git a/koha-tmpl/intranet-tmpl/prog/css/src/staff-global.scss b/koha-tmpl/intranet-tmpl/prog/css/src/staff-global.scss index 49a5ce8767..ad1e61e48a 100644 --- a/koha-tmpl/intranet-tmpl/prog/css/src/staff-global.scss +++ b/koha-tmpl/intranet-tmpl/prog/css/src/staff-global.scss @@ -4646,6 +4646,39 @@ div .suggestion_note { white-space:nowrap } } +/* Elasticsearch autocomplete menu */ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + display: inline-block; +} + +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + /*position the autocomplete items to be the same width as the container:*/ + + div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + + &:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + + &.autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: lighten(#85CA11, 45%); + } + } +} +/* /Elasticsearch autocomplete menu */ #camera, #output { border: 8px solid #EDF4F6; diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/js_includes.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/js_includes.inc index d9f19b7229..f31fa4ef65 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/js_includes.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/js_includes.inc @@ -170,4 +170,8 @@ }); [% END %] + +[% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] +[% Asset.js("js/elasticsearch/autocomplete.js") | $raw %] +[% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index 69619fd6f8..2e9d0a6d3e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -298,3 +298,20 @@ Searching: - LIBRIS base URL - pref: LibrisURL - "Please only change this if you are sure it needs changing." + - + - pref: OPACAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for OPAC. + - + - pref: IntranetAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for Intranet. + - diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt index 29808258d1..7f7dc96725 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt @@ -382,6 +382,10 @@ var dad = line.parentNode; dad.appendChild(line.cloneNode(true)); line.removeChild(ButtonPlus); + /* Intranet JS IntranetAutocompleteElasticSearch */ + [% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] + AutocompleteInitIntranet(); + [% END %] } var Sticky; diff --git a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js b/koha-tmpl/intranet-tmpl/prog/js/staff-global.js index f2becd2a7f..1fd4987e4e 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js +++ b/koha-tmpl/intranet-tmpl/prog/js/staff-global.js @@ -89,6 +89,13 @@ $(document).ready(function() { $(".keep_text").on("click",function(){ var field_index = $(this).parent().index(); keep_text( field_index ); + /* IntranetAutocompleteElasticSearch Tab */ + var tab = this.hash.substr(1, this.hash.length-1); + /* Koha.Preference('IntranetAutocompleteElasticSearch') == Show */ + if (typeof AutocompleteInitIntranet !== "undefined" && tab === 'catalog_search' ){ + AutocompleteInitIntranet(); + } + $("#search-form").focus(); }); $(".toggle_element").on("click",function(e){ diff --git a/koha-tmpl/opac-tmpl/bootstrap/css/src/opac.scss b/koha-tmpl/opac-tmpl/bootstrap/css/src/opac.scss index 747251450f..e75fa9a3d8 100644 --- a/koha-tmpl/opac-tmpl/bootstrap/css/src/opac.scss +++ b/koha-tmpl/opac-tmpl/bootstrap/css/src/opac.scss @@ -2872,4 +2872,38 @@ $star-selected: #EDB867; } } +/* Elasticsearch autocomplete menu */ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + width: 100%; +} + +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 9999; + + div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + + &:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + + &.autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: lighten( $input-btn-focus-color, 45% ); + } + } +} + +/* /Elasticsearch autocomplete menu */ + @import "responsive"; diff --git a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc index 2d61b53590..dcd19b42dc 100644 --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc +++ b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc @@ -289,6 +289,10 @@ $(document).ready(function() { }); [% PROCESS jsinclude %] + +[% IF ( Koha.Preference('OPACAutocompleteElasticSearch') ) %] + [% Asset.js("js/opac-elasticsearch/opac-autocomplete.js") | $raw %] +[% END %] [% IF ( Koha.Preference('OPACUserJS') ) %] diff --git a/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js b/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js new file mode 100644 index 0000000000..ef2bdbd21e --- /dev/null +++ b/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js @@ -0,0 +1,279 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +var url_request = '/cgi-bin/koha/svc/elasticsearch/opac-elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitOpac(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + autocomplete(input_q[nb], nb, left, right, top); + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + $(div_relative).css("display", "inline-block"); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + var a, val = this.value; + /* var for async compare */ + var tmp_input = this.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", this.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + this.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + /* autocomplete for all prefix */ + if (chose_prefix === 'kw' || chose_prefix === ''){ + myset = GetSetAutocompleteAllIdx(data, prefix, key); + } else { // autocomplete for one prefix + myset = GetSetAutocompleteOneIdx(data, prefix, key); + }; + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(this); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for only one prefix title/author/etc... */ + function GetSetAutocompleteOneIdx(data, prefix, key){ + let myset = new Set(); + let tmp_data = data['hits']['hits']; + for (let i = 0; i < tmp_data.length; i++) { + for (let j = 0; j < tmp_data[i]['highlight'][prefix + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][prefix + '.' + key][j]; + tmp = tmp.replace(/^\[/g, ''); + tmp = tmp.replace(/\]+$/g, ''); + myset.add(tmp.replace(/^[ &\/\\#,+)$~%.'":*?>{}!;]+|[ &\/\\#,+($~%.'":*?<{}!;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + return myset; + }; + /* get autocomplete for all prefix */ + function GetSetAutocompleteAllIdx(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#booleansearch input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitOpac(); diff --git a/opac/svc/elasticsearch/opac-elasticsearch.pl b/opac/svc/elasticsearch/opac-elasticsearch.pl new file mode 100755 index 0000000000..ad1c4b4568 --- /dev/null +++ b/opac/svc/elasticsearch/opac-elasticsearch.pl @@ -0,0 +1,162 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +use Koha::Items; +use C4::Context; +use C4::Biblio qw ( GetMarcBiblio ); + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +my $iskey = 0; + +if ($session->param("key") eq "autocomplete") { + $iskey = 1; + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + #search by many prefix fields + if ($length > 1){ + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + + if (C4::Context->preference('OpacSuppression') || C4::Context->yaml_preference('OpacHiddenItems')) { + my @prefix = $res->{ "prefix" }; + @prefix = split(',', $prefix[0]); + + for (my $i = 0; $i < scalar @prefix; $i++) { + filterAutocomplete($res->{ $i }->{ 'hits' }); + } + } + + print $cgi->header("application/json"); + print to_json($res); + } + #search by one prefix field + elsif ($length == 1) { + my $res = $browser->autocomplete_one_idx($ses, $prefix[0], $session->param("analyzer"), $session->param("token_counter")); + + if (C4::Context->preference('OpacSuppression') || C4::Context->yaml_preference('OpacHiddenItems')) { + filterAutocomplete($res->{ 'hits' }); + } + + print $cgi->header("application/json"); + print to_json($res); + } + #no prefix 404 + else { + response404JSON(); + } +} + +if ($iskey == 0) { + response404JSON(); +} + +sub filterAutocomplete { + my $hits = $_[0]; + my $hitlist = $hits->{ "hits" }; + if (@{$hitlist}) { + # Remove item inside hits in elasticsearch response if the item has + # marc field 942$n set to true and OpacSuppression preference on + if (C4::Context->preference('OpacSuppression')) { + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $record = GetMarcBiblio({ + biblionumber => $hitlist->[$i]->{ "_id" }, + opac => 1 + }); + my $opacsuppressionfield = '942'; + my $opacsuppressionfieldvalue = $record->field($opacsuppressionfield); + if ( $opacsuppressionfieldvalue && + $opacsuppressionfieldvalue->subfield("n") && + $opacsuppressionfieldvalue->subfield("n") == 1) { + # if OPAC suppression by IP address + if (C4::Context->preference('OpacSuppressionByIPRange')) { + my $IPAddress = $ENV{'REMOTE_ADDR'}; + my $IPRange = C4::Context->preference('OpacSuppressionByIPRange'); + if ($IPAddress !~ /^$IPRange/) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } else { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + } + # Remove item inside hits in elasticsearch response if the item is + # declared hidden in OPACHiddenItems preference + if (C4::Context->yaml_preference('OpacHiddenItems')) { + my @biblionumbers; + foreach (@{$hitlist}) { + push(@biblionumbers, $_->{ "_id" }); + } + my $autocomplete_items = Koha::Items->search({ + biblionumber => { -in => \@biblionumbers } + }); + my $filtered_items = $autocomplete_items->filter_by_visible_in_opac({ + patron => undef + }); + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $item = $filtered_items->find({ + biblionumber => $hitlist->[$i]->{ "_id" } + }); + if (!$item) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + # Adjust the max_score inside hits in elasticsearch response + my $maxscore = 0; + foreach ( @{$hitlist} ) { + my $score = $_->{"_score"}; + $maxscore = $score if ($maxscore < $score); + } + if ($maxscore == 0) { + $hits->{ "max_score" } = undef; + } else { + $hits->{ "max_score" } = $maxscore; + } + } +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} diff --git a/t/Koha_SearchEngine_Elasticsearch_Browse.t b/t/Koha_SearchEngine_Elasticsearch_Browse.t index 986c4ba658..4e93e2c42d 100755 --- a/t/Koha_SearchEngine_Elasticsearch_Browse.t +++ b/t/Koha_SearchEngine_Elasticsearch_Browse.t @@ -65,4 +65,34 @@ subtest "_build_query tests" => sub { }, 'Fuzziness and size specified'); }; +subtest "_build_query_autocomplete tests" => sub { + plan tests => 1; + + my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'}); + + my $q = $browse->_build_query_autocomplete('a', 'title', 'autocomplete'); + + is_deeply($q, { + _source => ["title"], + query => { + match => { + "title.autocomplete" => { + query => 'a', + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => 1, + fragment_size => 100, + pre_tags => [""], + post_tags => [""], + fields => { + "title.autocomplete" => {} + } + } + }, 'Autocomplete for title is specified'); + +}; + done_testing(); -- 2.25.1