@@ -, +, @@ - DEV atomic update: bug_27113-elasticsearch_autocomplete_input_search.perl - Upgrade to XXX done : Bug 27113 - Autocomplete input on main page with elasticsearch - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; - IntranetAutocompleteElasticSearch; - OPACAutocompleteElasticSearch; --- Koha/SearchEngine/Elasticsearch/Browse.pm | 127 +++++++- .../elasticsearch/field_config.yaml | 4 + .../elasticsearch/index_config.yaml | 14 + api/elasticsearch/elasticsearch.pl | 58 ++++ ...asticsearch_autocomplete_input_search.perl | 10 + installer/data/mysql/mandatory/sysprefs.sql | 2 + .../js/elasticsearch/autocomplete.js | 270 ++++++++++++++++++ .../prog/css/elasticsearch/autocomplete.css | 28 ++ .../prog/en/includes/doc-head-close.inc | 5 + .../prog/en/includes/js_includes.inc | 4 + .../modules/admin/preferences/searching.pref | 17 ++ .../prog/en/modules/catalogue/advsearch.tt | 4 + .../intranet-tmpl/prog/js/staff-global.js | 7 + .../opac-elasticsearch/opac-autocomplete.css | 29 ++ .../bootstrap/en/includes/doc-head-close.inc | 5 + .../bootstrap/en/includes/opac-bottom.inc | 4 + .../bootstrap/en/modules/opac-advsearch.tt | 8 + .../opac-elasticsearch/opac-autocomplete.js | 266 +++++++++++++++++ opac/svc/elasticsearch/opac-elasticsearch.pl | 145 ++++++++++ t/Koha_SearchEngine_Elasticsearch_Browse.t | 30 ++ 20 files changed, 1036 insertions(+), 1 deletion(-) create mode 100755 api/elasticsearch/elasticsearch.pl create mode 100644 installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl create mode 100644 koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js create mode 100644 koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css create mode 100644 koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css create mode 100644 koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js create mode 100755 opac/svc/elasticsearch/opac-elasticsearch.pl --- a/Koha/SearchEngine/Elasticsearch/Browse.pm +++ a/Koha/SearchEngine/Elasticsearch/Browse.pm @@ -40,7 +40,7 @@ Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch This provides an easy interface to the "browse" functionality. Essentially, it does a fast prefix search on defined fields. The fields have to be marked -as "suggestible" in the database when indexing takes place. +as "suggestible" in the database when indexing takes place(no action required for autocomplete). =head1 METHODS @@ -162,6 +162,129 @@ sub _build_query { return $query; } +=head2 autocomplete_one_idx + + my $query = $self->autocomplete_one_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (only one prefix), looking for C<$cgi_q> , using analyzer C<$analyzer> , +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref of hashrefs with highlights. Each hashref contains a "text" element that contains the field as returned. + +=cut + +sub autocomplete_one_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my @source; + my $elasticsearch = $self->get_elasticsearch(); + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + my $res = $elasticsearch->search( + index => $self->index_name, + body => $query + ); + $res->{'val'} = $cgi_q; + $res->{'prefix'} = $prefix; + $res->{'token_counter'} = $token_counter; + + return $res; +} + +=head2 autocomplete_idx + + my $query = $self->autocomplete_idx($cgi_q, $prefix, $analyzer, $token_counter); + +Does a prefix search for C<$prefix> (many prefix), looking for C<$cgi_q>, using analyzer C<$analyzer>, +C<$token_counter> is used for identify which word to use in autocomplete + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub autocomplete_idx { + my ($self, $cgi_q, $prefix, $analyzer, $token_counter) = @_; + my %results; + my $idx = 0; + foreach my $pref ( @$prefix ) { + $results{$idx} = $self->autocomplete_one_idx($cgi_q, $pref, $analyzer, $token_counter); + $idx++; + } + $results{'val'} = $cgi_q; + $results{'prefix'} = join( ',', @$prefix ); + $results{'token_counter'} = $token_counter; + return \%results; +} + +=head2 _build_query_autocomplete + + my $query = $self->_build_query_autocomplete($cgi_q, $prefix, $analyzer); + +Arguments: + +=over 4 + +=item cgi_q + +GET request + +=item prefix + +Field(s) for autocomplete (title, author, etc...) + +=item analyzer + +Name of analyzer wich we use for autocomplete + +=back + +=cut + +=head3 Returns + +This returns an arrayref for all prefix of hashrefs with highlights. Each hashref contains a "text" element +that contains the field as returned. + +=cut + +sub _build_query_autocomplete { + my ($self, $cgi_q, $prefix, $analyzer) = @_; + my (@source); + #prefix + analyzer + my $prefix_analyzer = $prefix . '.' . $analyzer; + # we can change these variables + my ($nb_fragments, $size_fragment, $pre_tags, $post_tags) = (1, 100, [""], [""]); + push(@source, $prefix); + my $query = { + _source => \@source, + query => { + match => { + $prefix_analyzer => { + query => $cgi_q, + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => $nb_fragments, + fragment_size => $size_fragment, + pre_tags => $pre_tags, + post_tags => $post_tags, + fields => { + $prefix_analyzer => {} + } + } + }; + return $query; +} + 1; __END__ @@ -172,6 +295,8 @@ __END__ =item Robin Sheat << >> +=item Ivan Dziuba << >> + =back =cut --- a/admin/searchengine/elasticsearch/field_config.yaml +++ a/admin/searchengine/elasticsearch/field_config.yaml @@ -47,6 +47,10 @@ search: type: text analyzer: analyzer_phrase search_analyzer: analyzer_phrase + autocomplete: + type: text + analyzer: autocomplete + search_analyzer: standard raw: type: keyword normalizer: nfkc_cf_normalizer --- a/admin/searchengine/elasticsearch/index_config.yaml +++ a/admin/searchengine/elasticsearch/index_config.yaml @@ -2,6 +2,14 @@ # Index configuration that defines how different analyzers work. index: analysis: + tokenizer: + autocomplete_tokenizer: + type: edge_ngram + min_gram: 1 + max_gram: 16 + token_chars: + - letter + - digit analyzer: # Phrase analyzer is used for phrases (exact phrase match) analyzer_phrase: @@ -10,6 +18,12 @@ index: - icu_folding char_filter: - punctuation + autocomplete: + type: custom + filter: + - icu_folding + - lowercase + tokenizer: autocomplete_tokenizer analyzer_standard: tokenizer: icu_tokenizer filter: --- a/api/elasticsearch/elasticsearch.pl +++ a/api/elasticsearch/elasticsearch.pl @@ -0,0 +1,58 @@ +#!/usr/bin/perl + +use Modern::Perl; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +if ($session->param("key") eq "autocomplete") { + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + if ($length >= 1) { + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + print $cgi->header("application/json;charset=UTF-8"); + print to_json($res, {utf8 => 1}); + } + #no prefix 404 + else { + response404JSON(); + } +} else { + response404JSON(); +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json;charset=UTF-8"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} --- a/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl +++ a/installer/data/mysql/atomicupdate/bug_27113-elasticsearch_autocomplete_input_search.perl @@ -0,0 +1,10 @@ +$DBversion = 'XXX'; # will be replaced by the RM +if( CheckVersion( $DBversion ) ) { + # you can use $dbh here like: + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('OPACAutocompleteElasticSearch', '0', NULL, 'If ON, show search suggestions in the OPAC when using Elasticsearch.', 'YesNo')}); + + $dbh->do(q{INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES ('IntranetAutocompleteElasticSearch', '0', NULL, 'If ON, show search suggestions in the staff interface when using Elasticsearch.', 'YesNo')}); + + # Always end with this (adjust the bug info) + NewVersion( $DBversion, 27113, "Autocomplete with Elasticsearch"); +} --- a/installer/data/mysql/mandatory/sysprefs.sql +++ a/installer/data/mysql/mandatory/sysprefs.sql @@ -286,6 +286,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('OnSiteCheckouts','0','','Enable/Disable the on-site checkouts feature','YesNo'), ('OnSiteCheckoutsForce','0','','Enable/Disable the on-site for all cases (Even if a user is debarred, etc.)','YesNo'), ('OnSiteCheckoutAutoCheck','0','','Enable/Do not enable onsite checkout by default if last checkout was an onsite checkout','YesNo'), +('IntranetAutocompleteElasticSearch','0',NULL,'If ON, show search suggestions in the staff interface when using Elasticsearch.','YesNo'), ('IntranetBiblioDefaultView','normal','normal|marc|isbd|labeled_marc','Choose the default detail view in the staff interface; choose between normal, labeled_marc, marc or isbd','Choice'), ('intranetbookbag','1','','If ON, enables display of Cart feature in the intranet','YesNo'), ('IntranetCirculationHomeHTML', '', NULL, 'Show the following HTML in a div on the bottom of the reports home page', 'Free'), @@ -407,6 +408,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('OPACAllowUserToChooseBranch','1','1','Allow the user to choose the branch they want to pickup their hold from','YesNo'), ('OPACAmazonCoverImages','0','','Display cover images on OPAC from Amazon Web Services','YesNo'), ('OpacAuthorities','1',NULL,'If ON, enables the search authorities link on OPAC','YesNo'), +('OPACAutocompleteElasticSearch','0',NULL,'If ON, show search suggestions in the OPAC when using Elasticsearch.','YesNo'), ('OPACBaseURL','',NULL,'Specify the Base URL of the OPAC, e.g., http://opac.mylibrary.com, including the protocol (http:// or https://). Otherwise, the http:// will be added automatically by Koha upon saving.','Free'), ('opacbookbag','1','','If ON, enables display of Cart feature','YesNo'), ('OpacBrowser','0',NULL,'If ON, enables subject authorities browser on OPAC (needs to set misc/cronjob/sbuild_browser_and_cloud.pl)','YesNo'), --- a/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js +++ a/koha-tmpl/intranet-tmpl/js/elasticsearch/autocomplete.js @@ -0,0 +1,270 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +/* stop class for elements name=["q"] */ +var stop_class_input = ["form-field-value"]; + +var url_request = '/cgi-bin/koha/api/elasticsearch/elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitIntranet(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + if (!stop_class_input.includes(input_q[nb].className)){ + autocomplete(input_q[nb], nb, left, right, top); + } + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + + var timer; + var doneTimerInterval = 500; + + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + clearTimeout(timer); + timer = setTimeout( function() { + var a, val = inp.value; + /* var for async compare */ + var tmp_input = inp.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", inp.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + inp.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + contentType: "application/json;charset=UTF-8", + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + + myset = GetSetAutocomplete(data, prefix, key); + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(inp); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }, doneTimerInterval); + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for all prefix */ + function GetSetAutocomplete(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#advanced-search input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitIntranet(); --- a/koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css +++ a/koha-tmpl/intranet-tmpl/prog/css/elasticsearch/autocomplete.css @@ -0,0 +1,28 @@ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + display: inline-block; +} +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + /*position the autocomplete items to be the same width as the container:*/ + } + .autocomplete-items div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + } + .autocomplete-items div:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + .autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: #cedfb1 !important; + color: #ffffff; + } --- a/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc +++ a/koha-tmpl/intranet-tmpl/prog/en/includes/doc-head-close.inc @@ -26,6 +26,11 @@ [% Asset.css("css/print.css", { media = "print" }) | $raw %] [% INCLUDE intranetstylesheet.inc %] [% IF ( bidi ) %][% Asset.css("css/right-to-left.css") | $raw %][% END %] + +[% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] + [% SET Optylesheet = 'elasticsearch/autocomplete.css' %] + +[% END %] [% END %] + +[% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] +[% Asset.js("js/elasticsearch/autocomplete.js") | $raw %] +[% END %] --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -298,3 +298,20 @@ Searching: - LIBRIS base URL - pref: LibrisURL - "Please only change this if you are sure it needs changing." + - + - pref: OPACAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for OPAC. + - + - pref: IntranetAutocompleteElasticSearch + type: boolean + default: 0 + choices: + 1: Show + 0: "Don't show" + - looking terms based on a provided text by using an ElasticSearch for Intranet. + - --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/advsearch.tt @@ -382,6 +382,10 @@ var dad = line.parentNode; dad.appendChild(line.cloneNode(true)); line.removeChild(ButtonPlus); + /* Intranet JS IntranetAutocompleteElasticSearch */ + [% IF ( Koha.Preference('IntranetAutocompleteElasticSearch') ) %] + AutocompleteInitIntranet(); + [% END %] } var Sticky; --- a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js +++ a/koha-tmpl/intranet-tmpl/prog/js/staff-global.js @@ -89,6 +89,13 @@ $(document).ready(function() { $(".keep_text").on("click",function(){ var field_index = $(this).parent().index(); keep_text( field_index ); + /* IntranetAutocompleteElasticSearch Tab */ + var tab = this.hash.substr(1, this.hash.length-1); + /* Koha.Preference('IntranetAutocompleteElasticSearch') == Show */ + if (typeof AutocompleteInitIntranet !== "undefined" && tab === 'catalog_search' ){ + AutocompleteInitIntranet(); + } + $("#search-form").focus(); }); $(".toggle_element").on("click",function(e){ --- a/koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css +++ a/koha-tmpl/opac-tmpl/bootstrap/css/opac-elasticsearch/opac-autocomplete.css @@ -0,0 +1,29 @@ +/* CSS file OPACAutocompleteElasticSearch */ +.autocomplete { + /*the container must be positioned relative:*/ + position: relative; + display: inline-block; + width: 100%; +} +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + } + .autocomplete-items div { + padding: 10px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; + } + .autocomplete-items div:hover { + /*when hovering an item:*/ + background-color: #e9e9e9; + } + .autocomplete-active { + /*when navigating through the items using the arrow keys:*/ + background-color: #cedfb1 !important; + color: #ffffff; + } --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc +++ a/koha-tmpl/opac-tmpl/bootstrap/en/includes/doc-head-close.inc @@ -23,6 +23,11 @@ [% SET opaclayoutstylesheet = 'opac.css' %] [% END %] [% END %] + +[% IF ( Koha.Preference('OPACAutocompleteElasticSearch') ) %] + [% SET Optylesheet = 'opac-elasticsearch/opac-autocomplete.css' %] + +[% END %] [% IF (opaclayoutstylesheet.match('^https?:|^\/')) %] [% ELSE %] --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc +++ a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc @@ -289,6 +289,10 @@ $(document).ready(function() { }); [% PROCESS jsinclude %] + +[% IF ( Koha.Preference('OPACAutocompleteElasticSearch') ) %] + [% Asset.js("js/opac-elasticsearch/opac-autocomplete.js") | $raw %] +[% END %] [% IF ( Koha.Preference('OPACUserJS') ) %] --- a/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js +++ a/koha-tmpl/opac-tmpl/bootstrap/js/opac-elasticsearch/opac-autocomplete.js @@ -0,0 +1,266 @@ +/* OPAC JS file OPACAutocompleteElasticSearch */ +/* prefix for search ES */ +var es_prefix = { + 'au': 'author', + 'pb': 'publisher', + 'se': 'title-series', + 'su': 'subject', + 'ti': 'title-cover', + /* for all */ + '': ['title-cover', 'author', 'subject', 'title-series', 'publisher'], + 'kw': ['title-cover', 'author', 'subject', 'title-series', 'publisher'] +}; + +var url_request = '/cgi-bin/koha/svc/elasticsearch/opac-elasticsearch.pl?q='; +/* query for elasticsearch encode*/ +var query_url_encode = { + '\'':'%5C%27', /* \\' for decode */ + '+': '' +}; +/* query for elasticsearch decode*/ +var query_url_decode = { + '\'':"\\'", + '+': '' +}; +/* count of lines for autocomplete */ +var nb_autocomplete = 10; +/* key API */ +var key = 'autocomplete'; + +function AutocompleteInitOpac(){ + /* vars for class position absolute autocomplete */ + var left = "0px"; + var right = "0px"; + var top = ""; + /* get all input name q for search */ + var input_q = document.getElementsByName("q"); + for (var nb = 0; nb < input_q.length; nb++){ + /* addEventListener for every 'input' */ + autocomplete(input_q[nb], nb, left, right, top); + }; +}; + +function autocomplete(inp, nb, left, right) { + var select_idx = document.getElementsByName("idx"); + /* autocomplete off for input */ + inp.setAttribute("autocomplete", "off"); + /* get parent of input */ + var parent_inp = $(inp).parent(); + /* get element after input */ + var next_elem_inp = inp.nextElementSibling; + /* create new div with position relative for class .autocomplete with absolute */ + var div_relative = document.createElement('div'); + $(div_relative).addClass( "autocomplete" ); + $(div_relative).css("display", "inline-block"); + div_relative.append(inp); + /* input doesn't have an elem after, add it to parent */ + if (next_elem_inp === null){ + parent_inp.append( div_relative ); + } else { // input has an elem after, add elem after it + next_elem_inp.before(div_relative); + }; + var currentFocus; + + var timer; + var doneTimerInterval = 500; + + /*execute a function when someone writes in the text field:*/ + var token_counter = 0; + inp.addEventListener("input", function(e) { + clearTimeout(timer); + timer = setTimeout( function() { + var a, val = inp.value; + /* var for async compare */ + var tmp_input = inp.value.replace(/[+']/g, function(matched){ + return query_url_decode[matched]; + }); + token_counter++; + currentFocus = -1; + if (document.getElementsByClassName("autocomplete-items").length !== 0){ + a = document.getElementsByClassName("autocomplete-items")[0]; + } else { + /*create a DIV element that will contain the items (values):*/ + a = document.createElement("DIV"); + a.setAttribute("id", inp.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + /*append the DIV element as a child of the autocomplete container:*/ + inp.parentNode.appendChild(a); + /*append position absolute left/right:*/ + $(".autocomplete-items").css("left",left); + $(".autocomplete-items").css("right",right); + }; + /* get es_prefix key for builder */ + var chose_prefix = (select_idx == null || select_idx.length == 0) ? '' : GetValueIdx(select_idx, nb); + chose_prefix = chose_prefix.replace(/([^,])[,-]([^,].*)?$/, '$1'); + if (chose_prefix !== null){ + var prefix = es_prefix[chose_prefix].toString(); + val = val.replace(/[+']/g, function(matched){ + return query_url_encode[matched]; + }); + if (tmp_input == '' || tmp_input == null){ + closeAllLists(); + token_counter = 0; + } else { + $.ajax({ + type: 'GET', + url: url_request + val + '&key=' + key + '&prefix=' + prefix + '&token_counter=' + token_counter, + contentType: "application/json;charset=UTF-8", + success: function (data) { + //console.log(data); + if (data.length !== 0){ + var myset; //Set for Autocomplete unique + + myset = GetSetAutocomplete(data, prefix, key); + /* append set to autocomplete */ + if ( tmp_input + prefix == data['val'] + data['prefix'] && token_counter === parseInt(data['token_counter'], 10)){ + a.innerHTML = ""; + for (let item of myset){ + a.appendChild(CreateDivItemAutocomplete(item, val)); + }; + }; + } else { + closeAllLists(inp); + }; + }, + error: function (data) { + console.log(data); + }, + }); + } + + }; + }, doneTimerInterval); + }); + /* get value for tag with name idx */ + function GetValueIdx(elem, nb){ + switch (elem[0].tagName){ + case 'INPUT': + return elem[0].value; + case 'SELECT': + return select_idx[nb].options[select_idx[nb].selectedIndex].value; + default: + return null; + }; + }; + /* get autocomplete for all prefix */ + function GetSetAutocomplete(data, prefix, key){ + let myset = new Set(); + var pref = prefix.split(","); + for (k = 0; k < Object.keys(data).length; k++){ //Object.keys(data).length + if (data[k] != '' && data[k] != null){ + let tmp_data = data[k]['hits']['hits']; + for (i = 0; i < tmp_data.length; i++) { + for (j = 0; j < tmp_data[i]['highlight'][pref[k] + '.' + key].length; j++){ + /* div with data for autocomplete */ + let tmp = tmp_data[i]['highlight'][pref[k] + '.' + key][j] + myset.add(tmp.replace(/[ &#,+()$~%.'":*?<{}!/;]+$/g, '')); + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + }; + if (myset.size >= nb_autocomplete) break; + } + } + return myset; + }; + + /*execute a function presses a key on the keyboard:*/ + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { //DOWN + /*If the arrow DOWN key is pressed, + increase the currentFocus variable:*/ + currentFocus++; + /*and and make the current item more visible:*/ + addActive(x); + } else if (e.keyCode == 38) { //up + /*If the arrow UP key is pressed, + decrease the currentFocus variable:*/ + currentFocus--; + /*and and make the current item more visible:*/ + addActive(x); + e.preventDefault(); + } else if (e.keyCode == 13) { + /*If the ENTER key is pressed, prevent the form from being submitted,*/ + //e.preventDefault(); + if (currentFocus > -1) { + /*and simulate a click on the "active" item:*/ + if (x) x[currentFocus].click(); + } + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 27) { + closeAllLists(); + } + /* press Esc clear all autocomplete */ + else if (e.keyCode == 8) { + closeAllLists(); + } + /* press Tab clear all autocomplete */ + else if (e.keyCode == 9) { + closeAllLists(); + }; + }); + function addActive(x) { + /*a function to classify an item as "active":*/ + if (!x) return false; + /*start by removing the "active" class on all items:*/ + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + /*add class "autocomplete-active":*/ + x[currentFocus].classList.add("autocomplete-active"); + inp.value = (x[currentFocus].textContent.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + }; + function removeActive(x) { + /*a function to remove the "active" class from all autocomplete items:*/ + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + }; + }; + + function closeAllLists(elem) { + /*close all autocomplete lists in the document with class autocomplete-items */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]) + }; + }; + + /* div for one item autocomplete */ + function CreateDivItemAutocomplete (elem){ + var b = document.createElement("DIV"); + // add element "; + b.innerHTML += elem; + /*insert a input field that will hold the current array item's value:*/ + b.innerHTML += ""; + /*execute a function when someone clicks on the item value (DIV element):*/ + b.addEventListener("click", function(e) { + /* insert the value for the autocomplete text field: */ + inp.value = this.getElementsByTagName("input")[0].value; + /* normalyzer hightlight without tags */ + //inp.value = (inp.value.replace(/<\/?[^>]+(>|$)/g, "")).trim(); + inp.value = this.innerText; + + var autocommit = 1; + const inputs = document.querySelectorAll("#booleansearch input[type='text']"); + for (var i = 0; i < inputs.length && autocommit; i++) { + var input = inputs[i]; + if (input === inp) { + autocommit = 0; + } + } + //Submit form click mouse in div if not in advanced search + if (autocommit) this.closest("form").submit(); + }); + return b; + }; + + /*execute a function when someone clicks in the document:*/ + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +}; + +AutocompleteInitOpac(); --- a/opac/svc/elasticsearch/opac-elasticsearch.pl +++ a/opac/svc/elasticsearch/opac-elasticsearch.pl @@ -0,0 +1,145 @@ +#!/usr/bin/perl + +use Modern::Perl; +use CGI qw ( -utf8 ); +use JSON; +use utf8; +use Unicode::Normalize; +use CGI::Session; +use Koha::SearchEngine::Elasticsearch::Browse; + +use Koha::Items; +use C4::Context; +use C4::Biblio qw ( GetMarcBiblio ); + +my $browser = Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } ); +my $cgi = CGI->new; +my $session = CGI::Session->load() or die CGI::Session->errstr(); + +$session->param(-name=>'analyzer', -value=>"autocomplete"); +$session->param(-name=>'prefix', -value=>$cgi->multi_param("prefix")); +$session->param(-name=>'q', -value=>$cgi->multi_param("q")); +$session->param(-name=>'key', -value=>$cgi->multi_param("key")); +$session->param(-name=>'token_counter', -value=>$cgi->multi_param("token_counter")); +$session->expire('+1h'); + +if ($session->param("key") eq "autocomplete") { + my @prefix = split /,/, $session->param("prefix"); + #fields for autocomplete + my $length = scalar @prefix; + my $ses = NFKD( $session->param("q") ); + $ses =~ s/\p{NonspacingMark}//g; + + if ($length >= 1) { + my $res = $browser->autocomplete_idx($ses, \@prefix, $session->param("analyzer"), $session->param("token_counter")); + + filterAutocomplete($res); + + print $cgi->header("application/json;charset=UTF-8"); + print to_json($res, {utf8 => 1}); + } + #no prefix 404 + else { + response404JSON(); + } +} else { + response404JSON(); +} + +sub filterAutocomplete { + if (C4::Context->preference('OpacSuppression') || C4::Context->yaml_preference('OpacHiddenItems')) { + my $res = shift; + my @prefix = $res->{ "prefix" }; + @prefix = split(',', $prefix[0]); + + for (my $i = 0; $i < scalar @prefix; $i++) { + my $hits = $res->{ $i }->{ 'hits' }; + my $hitlist = $hits->{ "hits" }; + if (@{$hitlist}) { + # Remove item inside hits in elasticsearch response if the item has + # marc field 942$n set to true and OpacSuppression preference on + if (C4::Context->preference('OpacSuppression')) { + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $record = GetMarcBiblio({ + biblionumber => $hitlist->[$i]->{ "_id" }, + opac => 1 + }); + my $opacsuppressionfield = '942'; + my $opacsuppressionfieldvalue = $record->field($opacsuppressionfield); + if ( $opacsuppressionfieldvalue && + $opacsuppressionfieldvalue->subfield("n") && + $opacsuppressionfieldvalue->subfield("n") == 1) { + # if OPAC suppression by IP address + if (C4::Context->preference('OpacSuppressionByIPRange')) { + my $IPAddress = $ENV{'REMOTE_ADDR'}; + my $IPRange = C4::Context->preference('OpacSuppressionByIPRange'); + if ($IPAddress !~ /^$IPRange/) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } else { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + } + # Remove item inside hits in elasticsearch response if the item is + # declared hidden in OPACHiddenItems preference + if (C4::Context->yaml_preference('OpacHiddenItems')) { + my @biblionumbers; + foreach (@{$hitlist}) { + push(@biblionumbers, $_->{ "_id" }); + } + my $autocomplete_items = Koha::Items->search({ + biblionumber => { -in => \@biblionumbers } + }); + my $filtered_items = $autocomplete_items->filter_by_visible_in_opac({ + patron => undef + }); + for ( my $i = 0; $i < scalar @{$hitlist}; $i++ ) { + my $item = $filtered_items->find({ + biblionumber => $hitlist->[$i]->{ "_id" } + }); + if (!$item) { + splice(@{$hitlist}, $i, 1); + $i--; + $hits->{ "total" }--; + } + } + } + # Adjust the max_score inside hits in elasticsearch response + my $maxscore = 0; + foreach ( @{$hitlist} ) { + my $score = $_->{"_score"}; + $maxscore = $score if ($maxscore < $score); + } + if ($maxscore == 0) { + $hits->{ "max_score" } = undef; + } else { + $hits->{ "max_score" } = $maxscore; + } + } + } + } +} + +sub response404JSON { + my $res = CGI->new; + my $json = JSON->new->utf8; + my $header_type = "application/json;charset=UTF-8"; + my $header_status = "404"; + my $output = $json->encode({ + "error" => "No data", + "description" => "Bad request", + }); + print $res->header( + -type => $header_type, + -charset => "utf-8", + -status => $header_status + ); + print $output; + print "\n"; +} --- a/t/Koha_SearchEngine_Elasticsearch_Browse.t +++ a/t/Koha_SearchEngine_Elasticsearch_Browse.t @@ -65,4 +65,34 @@ subtest "_build_query tests" => sub { }, 'Fuzziness and size specified'); }; +subtest "_build_query_autocomplete tests" => sub { + plan tests => 1; + + my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'}); + + my $q = $browse->_build_query_autocomplete('a', 'title', 'autocomplete'); + + is_deeply($q, { + _source => ["title"], + query => { + match => { + "title.autocomplete" => { + query => 'a', + operator => 'and' + } + } + }, + highlight => { + number_of_fragments => 1, + fragment_size => 100, + pre_tags => [""], + post_tags => [""], + fields => { + "title.autocomplete" => {} + } + } + }, 'Autocomplete for title is specified'); + +}; + done_testing(); --