View | Details | Raw Unified | Return to bug 14567
Collapse All | Expand All

(-)a/Koha/Schema/Result/SearchField.pm (+6 lines)
Lines 119-122 __PACKAGE__->has_many( Link Here
119
119
120
__PACKAGE__->many_to_many("search_marc_maps", "search_marc_to_fields", "search_marc_map");
120
__PACKAGE__->many_to_many("search_marc_maps", "search_marc_to_fields", "search_marc_map");
121
121
122
123
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15
124
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pd5chYwH+KRhI2O8/e7bSg
125
126
127
# You can replace this text with custom code or comments, and it will be preserved on regeneration
122
1;
128
1;
(-)a/Koha/Schema/Result/SearchMarcMap.pm (+6 lines)
Lines 127-130 __PACKAGE__->has_many( Link Here
127
127
128
__PACKAGE__->many_to_many("search_fields", "search_marc_to_fields", "search_field");
128
__PACKAGE__->many_to_many("search_fields", "search_marc_to_fields", "search_field");
129
129
130
131
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2015-07-24 15:59:15
132
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/6VCm/kMfCjtmD4Kx3HGMA
133
134
135
# You can replace this text with custom code or comments, and it will be preserved on regeneration
130
1;
136
1;
(-)a/Koha/SearchEngine/Elasticsearch/Browse.pm (+184 lines)
Line 0 Link Here
1
package Koha::SearchEngine::Elasticsearch::Browse;
2
3
# Copyright 2015 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
=head1 NAME
21
22
Koha::SearchEngine::ElasticSearch::Browse - browse functions for Elasticsearch
23
24
=head1 SYNOPSIS
25
26
    my $browser =
27
      Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } );
28
    my $results = $browser->browse(
29
        'prefi', 'title',
30
        {
31
            results   => '500',
32
            fuzziness => 2,
33
        }
34
    );
35
    foreach my $r (@$results) {
36
        push @hits, $r->{text};
37
    }
38
39
=head1 DESCRIPTION
40
41
This provides an easy interface to the "browse" functionality. Essentially,
42
it does a fast prefix search on defined fields. The fields have to be marked
43
as "suggestible" in the database when indexing takes place.
44
45
=head1 METHODS
46
47
=cut
48
49
use base qw(Koha::ElasticSearch);
50
use Modern::Perl;
51
52
use Catmandu::Store::ElasticSearch;
53
54
use Carp;
55
use Data::Dumper;
56
57
Koha::SearchEngine::Elasticsearch::Browse->mk_accessors(qw( store ));
58
59
=head2 browse
60
61
    my $results = $browser->browse($prefix, $field, \%options);
62
63
Does a prefix search for C<$prefix>, looking in C<$field>. Options are:
64
65
=over 4
66
67
=item count
68
69
The number of results to return. For Koha browse purposes, this should
70
probably be fairly high. Defaults to 500.
71
72
=item fuzziness
73
74
How much allowing for typos and misspellings is done. If 0, then it must match
75
exactly. If unspecified, it defaults to '1', which is probably the most useful.
76
Otherwise, it is a number specifying the Levenshtein edit distance relative to
77
the string length, according to the following lengths:
78
79
=over 4
80
81
=item 0..2
82
83
must match exactly
84
85
=item 3..5
86
87
C<fuzziness> edits allowed
88
89
=item >5
90
91
C<fuzziness>+1 edits allowed
92
93
=back
94
95
In all cases the maximum number of edits allowed is two (an elasticsearch
96
restriction.)
97
98
=head3 Returns
99
100
This returns an arrayref of hashrefs. Each hashref contains a "text" element
101
that contains the field as returned. There may be other fields in that
102
hashref too, but they're less likely to be important.
103
104
The array will be ordered as returned from Elasticsearch, which seems to be
105
in order of some form of relevance.
106
107
=cut
108
109
sub browse {
110
    my ($self, $prefix, $field, $options) = @_;
111
112
    my $params = $self->get_elasticsearch_params();
113
    $self->store(
114
        Catmandu::Store::ElasticSearch->new(
115
            %$params,
116
        )
117
    ) unless $self->store;
118
119
    my $query = $self->_build_query($prefix, $field, $options);
120
    my $results = $self->store->bag->search(%$query);
121
    return $results->{suggest}{suggestions}[0]{options};
122
}
123
124
=head2 _build_query
125
126
    my $query = $self->_build_query($prefix, $field, $options);
127
128
Arguments are the same as for L<browse>. This will return a query structure
129
for elasticsearch to use.
130
131
=cut
132
133
sub _build_query {
134
    my ( $self, $prefix, $field, $options ) = @_;
135
136
    $options = {} unless $options;
137
    my $f = $options->{fuzziness} // 1;
138
    my $l = length($prefix);
139
    my $fuzzie;
140
    if ( $l <= 2 ) {
141
        $fuzzie = 0;
142
    }
143
    elsif ( $l <= 5 ) {
144
        $fuzzie = $f;
145
    }
146
    else {
147
        $fuzzie = $f + 1;
148
    }
149
    $fuzzie = 2 if $fuzzie > 2;
150
151
    my $size = $options->{count} // 500;
152
    my $query = {
153
        # this is an annoying thing, if we set size to 0 it gets rewritten
154
        # to 10. There's a bug somewhere in one of the libraries.
155
        size    => 1,
156
        suggest => {
157
            suggestions => {
158
                text       => $prefix,
159
                completion => {
160
                    field => $field . '__suggestion',
161
                    size  => $size,
162
                    fuzzy => {
163
                        fuzziness => $fuzzie,
164
                    }
165
                }
166
            }
167
        }
168
    };
169
    return $query;
170
}
171
172
1;
173
174
__END__
175
176
=head1 AUTHOR
177
178
=over 4
179
180
=item Robin Sheat C<< <robin@catalyst.net.nz> >>
181
182
=back
183
184
=cut
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-4 / +97 lines)
Lines 51-57 use Data::Dumper; # TODO remove Link Here
51
51
52
=head2 build_query
52
=head2 build_query
53
53
54
    my $simple_query = $builder->build_query("hello", %options)
54
    my $query = $builder->build_query($search, %options);
55
56
C<$search> is an arrayref where each element is a hashref containing
57
a part of the search.
58
59
The hashref must contain a 'term', which is the string, and may contain other
60
things to adjust how the search is performed:
61
62
=over 4
63
64
=item field
65
66
the field to search on, as defined in the C<search_field> table. If
67
unspecified, all fields will be searched over.
68
69
=item exact
70
71
do an exact, literal, phrase match rather than a tokenised search. This will
72
also ignore things like truncation and fuzziness.
73
74
=back
75
76
The C<%options> may be:
77
78
=over 4
79
80
=item and_or
81
82
either "and" or "or", and determines how multiple terms are combined. Default
83
is "and."
84
85
=back
86
87
=head3 Returns
88
89
a query structure intended to be passed to elasticsearch.
90
91
=head3 Note
92
93
This is pretty limited, it's intended that more options be added as necessary
94
to support new functionality.
95
96
=head3 Note 2
97
98
This ultimately ought to be replaced with a query parser driver, but that code
99
is not documented so it'll take a while to figure out.
100
101
=cut
102
103
sub build_query {
104
    my ( $self, $search, %options ) = @_;
105
106
    my @match_parts;
107
108
    foreach my $s (@$search) {
109
        my ( $m_type, $f_suffix, $query_name );
110
111
        # This accounts for different naming of things based on our options
112
        if ( $s->{exact} ) {
113
            $m_type     = 'term';
114
            $f_suffix   = '.raw';
115
            $query_name = 'value';
116
        }
117
        else {
118
            $m_type     = 'match';
119
            $f_suffix   = '';
120
            $query_name = 'query';
121
        }
122
        my $field = $s->{field} // "_all";
123
        my $m = {
124
            $m_type => {
125
                "$s->{field}$f_suffix" => {
126
                    $query_name => $s->{term},
127
                },
128
            },
129
        };
130
        push @match_parts, $m;
131
    }
132
    my $query;
133
    if ( $options{and_or} && lc( $options{and_or} ) eq 'or' ) {
134
        $query->{bool}{should} = \@match_parts;
135
    }
136
    else {
137
        $query->{bool}{must} = \@match_parts;
138
    }
139
    return { query => $query };
140
}
141
142
=head2 build_string_query
143
144
    my $simple_query = $builder->build_string_query("hello", %options)
55
145
56
This will build a query that can be issued to elasticsearch from the provided
146
This will build a query that can be issued to elasticsearch from the provided
57
string input. This expects a lucene style search form (see
147
string input. This expects a lucene style search form (see
Lines 72-80 according to these values. Valid values for C<direction> are 'asc' and 'desc'. Link Here
72
162
73
=back
163
=back
74
164
165
Note: This is mostly for compatibility, ideally it shouldn't get used
166
much.
167
75
=cut
168
=cut
76
169
77
sub build_query {
170
sub build_string_query {
78
    my ( $self, $query, %options ) = @_;
171
    my ( $self, $query, %options ) = @_;
79
172
80
    my $stemming         = C4::Context->preference("QueryStemming")        || 0;
173
    my $stemming         = C4::Context->preference("QueryStemming")        || 0;
Lines 455-461 sub build_authorities_query_compat { Link Here
455
Converts the zebra-style sort index information into elasticsearch-style.
548
Converts the zebra-style sort index information into elasticsearch-style.
456
549
457
C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
550
C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
458
something that can be sent to L<build_query>.
551
something that can be sent to L<build_string_query>.
459
552
460
=cut
553
=cut
461
554
Lines 492-498 sub _convert_sort_fields { Link Here
492
Converts zebra-style search index notation into elasticsearch-style.
585
Converts zebra-style search index notation into elasticsearch-style.
493
586
494
C<@indexes> is an array of index names, as presented to L<build_query_compat>,
587
C<@indexes> is an array of index names, as presented to L<build_query_compat>,
495
and it returns something that can be sent to L<build_query>.
588
and it returns something that can be sent to L<build_string_query>.
496
589
497
B<TODO>: this will pull from the elasticsearch mappings table to figure out
590
B<TODO>: this will pull from the elasticsearch mappings table to figure out
498
types.
591
types.
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-6 / +37 lines)
Lines 74-93 an offset (i.e. the number of the record to start with), rather than a page. Link Here
74
74
75
Returns
75
Returns
76
76
77
=over 4
78
79
=item records
80
81
an array of hashrefs for each record. Each hashref contains C<marc>, which has
82
a MARC::Record instance, and C<id>, which is the biblionumber of this record.
83
84
=item facets
85
86
facet data from Elasticsearch
87
88
=item hits
89
90
the total number of hits for this search
91
77
=cut
92
=cut
78
93
79
sub search {
94
sub search {
80
    my ($self, $query, $page, $count, %options) = @_;
95
    my ( $self, $query, $page, $count, %options ) = @_;
81
96
82
    my $params = $self->get_elasticsearch_params();
97
    my $params = $self->get_elasticsearch_params();
83
    my %paging;
98
    my %paging;
99
84
    # 20 is the default number of results per page
100
    # 20 is the default number of results per page
85
    $paging{limit} = $count || 20;
101
    $paging{limit} = $count || 20;
102
86
    # ES/Catmandu doesn't want pages, it wants a record to start from.
103
    # ES/Catmandu doesn't want pages, it wants a record to start from.
87
    if (exists $options{offset}) {
104
    if ( exists $options{offset} ) {
88
        $paging{start} = $options{offset};
105
        $paging{start} = $options{offset};
89
    } else {
106
    }
90
        $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
107
    else {
108
        $page = ( !defined($page) || ( $page <= 0 ) ) ? 0 : $page - 1;
91
        $paging{start} = $page * $paging{limit};
109
        $paging{start} = $page * $paging{limit};
92
    }
110
    }
93
    $self->store(
111
    $self->store(
Lines 101-107 sub search { Link Here
101
    if ($@) {
119
    if ($@) {
102
        die $self->process_error($@);
120
        die $self->process_error($@);
103
    }
121
    }
104
    return $results;
122
    my @records;
123
    $results->each(
124
        sub {
125
            # The results come in an array for some reason
126
            my $marc_json = @_[0]->{record};
127
            my $marc      = $self->json2marc($marc_json);
128
            push @records, { marc => $marc, id => @_[0]->{_id} };
129
        }
130
    );
131
    return {
132
        records => \@records,
133
        facets  => $results->{facets},
134
        hits    => $results->total
135
    };
105
}
136
}
106
137
107
=head2 count
138
=head2 count
Lines 165-171 sub search_compat { Link Here
165
    # consumers of this expect a name-spaced result, we provide the default
196
    # consumers of this expect a name-spaced result, we provide the default
166
    # configuration.
197
    # configuration.
167
    my %result;
198
    my %result;
168
    $result{biblioserver}{hits} = $results->total;
199
    $result{biblioserver}{hits}    = $all_results->{hits};
169
    $result{biblioserver}{RECORDS} = \@records;
200
    $result{biblioserver}{RECORDS} = \@records;
170
    return (undef, \%result, $self->_convert_facets($results->{facets}, $expanded_facet));
201
    return (undef, \%result, $self->_convert_facets($results->{facets}, $expanded_facet));
171
}
202
}
(-)a/opac/opac-browse.pl (+136 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# This is a CGI script that handles the browse feature.
4
5
# Copyright 2015 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
use Modern::Perl;
23
use CGI qw( -utf8 :standard );
24
25
use C4::Auth;
26
use C4::Context;
27
28
use Koha::ElasticSearch;
29
use Koha::SearchEngine::Elasticsearch::Browse;
30
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
31
use Koha::SearchEngine::Elasticsearch::Search;
32
33
use JSON;
34
use Unicode::Collate;
35
36
my $query = new CGI;
37
binmode STDOUT, ':utf8';
38
39
# This is the temporary entrance point to the API. Once bug #13799 is settled,
40
# it should be ported to using that.
41
my $api = $query->param('api');
42
43
if ( !$api ) {
44
45
    # No parameters, so we just render the template.
46
47
}
48
elsif ( $api eq 'GetSuggestions' ) {
49
    my $fuzzie = $query->param('fuzziness');
50
    my $prefix = $query->param('prefix');
51
    my $field  = $query->param('field');
52
53
# Under a persistant environment, we should probably not reinit this every time.
54
    my $browser =
55
      Koha::SearchEngine::Elasticsearch::Browse->new( { index => 'biblios' } );
56
    my $res = $browser->browse( $prefix, $field, { fuzziness => $fuzzie } );
57
    my $collator = Unicode::Collate->new();
58
59
    # possible optimisation is to cache the collated values and do a straight
60
    # cmp.
61
    my @sorted = sort { $collator->cmp( $a->{text}, $b->{text} ) } @$res;
62
    print header(
63
        -type    => 'application/json',
64
        -charset => 'utf-8'
65
    );
66
    print to_json( \@sorted );
67
}
68
elsif ( $api eq 'GetResults' ) {
69
    my $term  = $query->param('term');
70
    my $field = $query->param('field');
71
72
    my $builder  = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
73
    my $searcher = Koha::SearchEngine::Elasticsearch::Search->new(
74
        { index => $Koha::ElasticSearch::BIBLIOS_INDEX } );
75
76
    my $query = $builder->build_query(
77
        [ { term => $term, field => $field, exact => 1 } ] );
78
    my $results = $searcher->search( $query, undef, 500 );
79
    my @output = _filter_for_output( $results->{records} );
80
    print header(
81
        -type    => 'application/json',
82
        -charset => 'utf-8'
83
    );
84
    print to_json( \@output );
85
}
86
87
# This is a temporary, MARC21-only thing that will grab titles, authors,
88
# and subjects. This should probably be done with some templatey gizmo
89
# in the future.
90
sub _filter_for_output {
91
    no warnings 'qw';    # it doesn't like the , in the qw
92
    my ($records) = @_;
93
94
    my @author_marc = qw(100,a 110,a 111,a 700,a 710,a 711,a);
95
    my @subjects_marc =
96
      qw(600 610 611 630 648 650 651);
97
    my @output;
98
    foreach my $rec (@$records) {
99
        my $marc = $rec->{marc};
100
        my $title;
101
        {
102
            # we do an ugly hack to fix up a little bit of bad formatting.
103
            my ($t_a, $t_b, $t_c) =
104
            (( $marc->subfield( 245, 'a' ) // '' ),
105
             ( $marc->subfield( 245, 'b' ) // '' ),
106
             ( $marc->subfield( 245, 'c' ) // '' ));
107
            $t_a .= ' ' unless $t_a =~ / $/;
108
            $t_b .= ' ' unless $t_b =~ / $/;
109
            $title = $t_a . $t_b . $t_c;
110
        }
111
        my @authors;
112
        foreach my $m (@author_marc) {
113
            push @authors, ( $marc->subfield( split( ',', $m ) ) // () );
114
        }
115
        my @subj_fields;
116
        foreach my $m (@subjects_marc) {
117
            # first collect all the fields
118
            push @subj_fields, $marc->field($m);
119
        }
120
        # Now grab the a and x from each field and reformat it
121
        my @subjects;
122
        foreach my $s (@subj_fields) {
123
            my @vals = (scalar($s->subfield('a')), $s->subfield('x'));
124
            next unless @vals;
125
            push @subjects, join( ' -- ', @vals );
126
        }
127
        push @output,
128
          {
129
            id => $rec->{id},
130
            title    => $title,
131
            authors  => \@authors,
132
            subjects => \@subjects,
133
          };
134
    }
135
    return @output;
136
}
(-)a/t/Koha_SearchEngine_Elasticsearch_Browse.t (-1 / +68 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2015 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Test::More;
23
24
use_ok('Koha::SearchEngine::Elasticsearch::Browse');
25
26
# testing browse itself not implemented as it'll require a running ES
27
can_ok('Koha::SearchEngine::Elasticsearch::Browse',
28
    qw/ _build_query browse /);
29
30
subtest "_build_query tests" => sub {
31
    plan tests => 2;
32
33
    my $browse = Koha::SearchEngine::Elasticsearch::Browse->new({index=>'dummy'});
34
    my $q = $browse->_build_query('foo', 'title');
35
    is_deeply($q, { size => 1,
36
        suggest => {
37
            suggestions => {
38
                text       => 'foo',
39
                completion => {
40
                    field => 'title__suggestion',
41
                    size  => 500,
42
                    fuzzy => {
43
                        fuzziness => 1,
44
                    }
45
                }
46
            }
47
        }
48
    }, 'No fuzziness or size specified');
49
50
    # Note that a fuzziness of 4 will get reduced to 2.
51
    $q = $browse->_build_query('foo', 'title', { fuzziness => 4, count => 400 });
52
    is_deeply($q, { size => 1,
53
        suggest => {
54
            suggestions => {
55
                text       => 'foo',
56
                completion => {
57
                    field => 'title__suggestion',
58
                    size  => 400,
59
                    fuzzy => {
60
                        fuzziness => 2,
61
                    }
62
                }
63
            }
64
        }
65
    }, 'Fuzziness and size specified');
66
};
67
68
done_testing();

Return to bug 14567