View | Details | Raw Unified | Return to bug 12478
Collapse All | Expand All

(-)a/C4/Biblio.pm (+10 lines)
Lines 38-43 use C4::Charset; Link Here
38
use C4::Linker;
38
use C4::Linker;
39
use C4::OAI::Sets;
39
use C4::OAI::Sets;
40
40
41
42
41
use vars qw($VERSION @ISA @EXPORT);
43
use vars qw($VERSION @ISA @EXPORT);
42
44
43
BEGIN {
45
BEGIN {
Lines 3412-3417 sub ModBiblioMarc { Link Here
3412
    $sth = $dbh->prepare("UPDATE biblioitems SET marc=?,marcxml=? WHERE biblionumber=?");
3414
    $sth = $dbh->prepare("UPDATE biblioitems SET marc=?,marcxml=? WHERE biblionumber=?");
3413
    $sth->execute( $record->as_usmarc(), $record->as_xml_record($encoding), $biblionumber );
3415
    $sth->execute( $record->as_usmarc(), $record->as_xml_record($encoding), $biblionumber );
3414
    $sth->finish;
3416
    $sth->finish;
3417
    if ( C4::Context->preference('SearchEngine') eq 'ElasticSearch' ) {
3418
# shift to its on sub, so it can do it realtime or queue
3419
        can_load( modules => { 'Koha::ElasticSearch::Indexer' => undef } );
3420
        # need to get this from syspref probably biblio/authority for index
3421
        my $indexer = Koha::ElasticSearch::Indexer->new();
3422
        my $records = [$record];
3423
        $indexer->update_index([$biblionumber], $records);
3424
    }
3415
    ModZebra( $biblionumber, "specialUpdate", "biblioserver" );
3425
    ModZebra( $biblionumber, "specialUpdate", "biblioserver" );
3416
    return $biblionumber;
3426
    return $biblionumber;
3417
}
3427
}
(-)a/C4/Search.pm (-2 / +8 lines)
Lines 2377-2385 sub _ZOOM_event_loop { Link Here
2377
    }
2377
    }
2378
}
2378
}
2379
2379
2380
=head2 new_record_from_zebra
2380
=head2 new_record_from_searchengine
2381
2381
2382
Given raw data from a Zebra result set, return a MARC::Record object
2382
Given raw data from a searchengine result set, return a MARC::Record object
2383
2383
2384
This helper function is needed to take into account all the involved
2384
This helper function is needed to take into account all the involved
2385
system preferences and configuration variables to properly create the
2385
system preferences and configuration variables to properly create the
Lines 2388-2393 MARC::Record object. Link Here
2388
If we are using GRS-1, then the raw data we get from Zebra should be USMARC
2388
If we are using GRS-1, then the raw data we get from Zebra should be USMARC
2389
data. If we are using DOM, then it has to be MARCXML.
2389
data. If we are using DOM, then it has to be MARCXML.
2390
2390
2391
If we are using elasticsearch, it'll already be a MARC::Record.
2392
2391
=cut
2393
=cut
2392
2394
2393
sub new_record_from_zebra {
2395
sub new_record_from_zebra {
Lines 2398-2403 sub new_record_from_zebra { Link Here
2398
    my $index_mode = ( $server eq 'biblioserver' )
2400
    my $index_mode = ( $server eq 'biblioserver' )
2399
                        ? C4::Context->config('zebra_bib_index_mode') // 'grs1'
2401
                        ? C4::Context->config('zebra_bib_index_mode') // 'grs1'
2400
                        : C4::Context->config('zebra_auth_index_mode') // 'dom';
2402
                        : C4::Context->config('zebra_auth_index_mode') // 'dom';
2403
    my $search_engine = C4::Context->preference("SearchEngine");
2404
    if ($search_engine eq 'Elasticsearch') {
2405
        return $raw_data;
2406
    }
2401
2407
2402
    my $marc_record =  eval {
2408
    my $marc_record =  eval {
2403
        if ( $index_mode eq 'dom' ) {
2409
        if ( $index_mode eq 'dom' ) {
(-)a/Koha/Biblio.pm (+105 lines)
Line 0 Link Here
1
package Koha::Biblio;
2
3
# This contains functions to do with managing biblio records.
4
5
# Copyright 2014 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
=head1 NAME
23
24
Koha::Biblio - contains fundamental biblio-related functions
25
26
=head1 DESCRIPTION
27
28
This contains functions for normal operations on biblio records.
29
30
Note: really, C4::Biblio does the main functions, but the Koha namespace is
31
the new thing that should be used.
32
33
=cut
34
35
use C4::Biblio; # EmbedItemsInMarcBiblio
36
use Koha::Biblio::Iterator;
37
use Koha::Database;
38
use Modern::Perl;
39
40
use base qw(Class::Accessor);
41
42
__PACKAGE__->mk_accessors(qw());
43
44
=head1 FUNCTIONS
45
46
=head2 get_all_biblios_iterator
47
48
    my $it = get_all_biblios_iterator();
49
50
This will provide an iterator object that will, one by one, provide the
51
MARC::Record of each biblio. This will include the item data.
52
53
The iterator is a Koha::Biblio::Iterator object.
54
55
=cut
56
57
sub get_all_biblios_iterator {
58
    my $database = Koha::Database->new();
59
    my $schema   = $database->schema();
60
    my $rs =
61
      $schema->resultset('Biblioitem')->search( { marc => { '!=', undef } },
62
        { columns => [qw/ biblionumber marc /] } );
63
    return Koha::Biblio::Iterator->new($rs, items => 1);
64
}
65
66
=head2 get_marc_biblio
67
68
    my $marc = get_marc_biblio($bibnum, %options);
69
70
This fetches the MARC::Record for the given biblio number. Nothing is returned
71
if the biblionumber couldn't be found (or it somehow has no MARC data.)
72
73
Options are:
74
75
=over 4
76
77
=item item_data
78
79
If set to true, item data is embedded in the record. Default is to not do this.
80
81
=back
82
83
=cut
84
85
sub get_marc_biblio {
86
    my ($class,$bibnum, %options) = @_;
87
88
    my $database = Koha::Database->new();
89
    my $schema   = $database->schema();
90
    my $rs =
91
      $schema->resultset('Biblioitem')
92
      ->search( { marc => { '!=', undef }, biblionumber => $bibnum },
93
        { columns => [qw/ marc /] } );
94
95
    my $row = $rs->next();
96
    return unless $row;
97
    my $marc = MARC::Record->new_from_usmarc($row->marc);
98
99
    # TODO implement this in this module
100
    C4::Biblio::EmbedItemsInMarcBiblio($marc, $bibnum) if $options{item_data};
101
102
    return $marc;
103
}
104
105
1;
(-)a/Koha/Biblio/Iterator.pm (+126 lines)
Line 0 Link Here
1
package Koha::Biblio::Iterator;
2
3
# This contains an iterator over biblio records
4
5
# Copyright 2014 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
=head1 NAME
23
24
Koha::Biblio::Iterator - iterates over biblios provided by a DBIx::Class::ResultSet
25
26
=head1 DESCRIPTION
27
28
This provides an iterator that gives the MARC::Record of each biblio that's
29
returned by a L<DBIx::Class::ResultSet> that provides a C<biblionumber>, and
30
C<marc> or C<marcxml> column from the biblioitems table.
31
32
=head1 SYNOPSIS
33
34
  use Koha::Biblio::Iterator;
35
  my $rs = $schema->resultset('biblioitems');
36
  my $iterator = Koha::Biblio::Iterator->new($rs);
37
  while (my $record = $iterator->next()) {
38
      // do something with $record
39
  }
40
41
=head1 METHODS
42
43
=cut
44
45
use C4::Biblio;    # :( - for EmbedItemsInMarcBiblio
46
47
use Carp;
48
use MARC::Record;
49
use MARC::File::XML;
50
use Modern::Perl;
51
52
=head2 new
53
54
    my $it = new($sth, option => $value, ...);
55
56
Takes a ResultSet to iterate over, and gives you an iterator on it. Optional
57
options may be specified.
58
59
=head3 Options
60
61
=over 4
62
63
=item items
64
65
Set to true to include item data in the resulting MARC record.
66
67
=back
68
69
=cut
70
71
sub new {
72
    my ( $class, $rs, %options ) = @_;
73
74
    bless {
75
        rs => $rs,
76
        %options,
77
    }, $class;
78
}
79
80
=head2 next()
81
82
In a scalar context, provides the next MARC::Record from the ResultSet, or
83
C<undef> if there are no more.
84
85
In a list context it will provide ($biblionumber, $record).
86
87
=cut
88
89
sub next {
90
    my ($self) = @_;
91
92
    my $marc;
93
    my $row = $self->{rs}->next();
94
    return if !$row;
95
    if ( $row->marc ) {
96
        $marc = MARC::Record->new_from_usmarc( $row->marc );
97
    }
98
    elsif ( $row->marcxml ) {
99
        $marc = MARC::Record->new_from_xml( $row->marcxml );
100
    }
101
    else {
102
        confess "No marc or marcxml column returned in the request.";
103
    }
104
105
    my $bibnum;
106
    if ( $self->{items} ) {
107
        $bibnum = $row->get_column('biblionumber');
108
        confess "No biblionumber column returned in the request."
109
          if ( !defined($bibnum) );
110
111
        # TODO this should really be in Koha::Biblio or something similar.
112
        C4::Biblio::EmbedItemsInMarcBiblio( $marc, $bibnum );
113
    }
114
115
    if (wantarray) {
116
        $bibnum //= $row->get_column('biblionumber');
117
        confess "No biblionumber column returned in the request."
118
          if ( !defined($bibnum) );
119
        return ( $bibnum, $marc );
120
    }
121
    else {
122
        return $marc;
123
    }
124
}
125
126
1;
(-)a/Koha/Database.pm (-60 / +7 lines)
Lines 40-45 use base qw(Class::Accessor); Link Here
40
40
41
__PACKAGE__->mk_accessors(qw( ));
41
__PACKAGE__->mk_accessors(qw( ));
42
42
43
our $schema; # the schema is a singleton
44
43
# _new_schema
45
# _new_schema
44
# Internal helper function (not a method!). This creates a new
46
# Internal helper function (not a method!). This creates a new
45
# database connection from the data given in the current context, and
47
# database connection from the data given in the current context, and
Lines 64-85 creates one, and connects to the database. Link Here
64
66
65
This database handle is cached for future use: if you call
67
This database handle is cached for future use: if you call
66
C<$database-E<gt>schema> twice, you will get the same handle both
68
C<$database-E<gt>schema> twice, you will get the same handle both
67
times. If you need a second database handle, use C<&new_schema> and
69
times. If you need a second database handle, use C<&new_schema>.
68
possibly C<&set_schema>.
69
70
70
=cut
71
=cut
71
72
72
sub schema {
73
sub schema {
73
    my $self = shift;
74
    my $self = shift;
74
    my $sth;
75
    my $sth;
75
    if ( defined( $self->{"schema"} ) && $self->{"schema"}->storage->connected() ) {
76
    if ( defined( $schema ) && $schema->storage->connected() ) {
76
        return $self->{"schema"};
77
        return $schema;
77
    }
78
    }
78
79
79
    # No database handle or it died . Create one.
80
    # No database handle or it died . Create one.
80
    $self->{"schema"} = &_new_schema();
81
    $schema = &_new_schema();
81
82
82
    return $self->{"schema"};
83
    return $schema;
83
}
84
}
84
85
85
=head2 new_schema
86
=head2 new_schema
Lines 102-161 sub new_schema { Link Here
102
    return &_new_schema();
103
    return &_new_schema();
103
}
104
}
104
105
105
=head2 set_schema
106
107
  $my_schema = $database->new_schema;
108
  $database->set_schema($my_schema);
109
  ...
110
  $database->restore_schema;
111
112
C<&set_schema> and C<&restore_schema> work in a manner analogous to
113
C<&set_context> and C<&restore_context>.
114
115
C<&set_schema> saves the current database handle on a stack, then sets
116
the current database handle to C<$my_schema>.
117
118
C<$my_schema> is assumed to be a good database handle.
119
120
=cut
121
122
sub set_schema {
123
    my $self       = shift;
124
    my $new_schema = shift;
125
126
    # Save the current database handle on the handle stack.
127
    # We assume that $new_schema is all good: if the caller wants to
128
    # screw himself by passing an invalid handle, that's fine by
129
    # us.
130
    push @{ $self->{"schema_stack"} }, $self->{"schema"};
131
    $self->{"schema"} = $new_schema;
132
}
133
134
=head2 restore_schema
135
136
  $database->restore_schema;
137
138
Restores the database handle saved by an earlier call to
139
C<$database-E<gt>set_schema>.
140
141
=cut
142
143
sub restore_schema {
144
    my $self = shift;
145
146
    if ( $#{ $self->{"schema_stack"} } < 0 ) {
147
148
        # Stack underflow
149
        die "SCHEMA stack underflow";
150
    }
151
152
    # Pop the old database handle and set it.
153
    $self->{"schema"} = pop @{ $self->{"schema_stack"} };
154
155
    # FIXME - If it is determined that restore_context should
156
    # return something, then this function should, too.
157
}
158
159
=head2 EXPORT
106
=head2 EXPORT
160
107
161
None by default.
108
None by default.
(-)a/Koha/ElasticSearch.pm (+324 lines)
Line 0 Link Here
1
package Koha::ElasticSearch;
2
3
# Copyright 2013 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use base qw(Class::Accessor);
21
22
use C4::Context;
23
use Carp;
24
use Elasticsearch;
25
use Koha::Database;
26
use Modern::Perl;
27
28
use Data::Dumper;    # TODO remove
29
30
__PACKAGE__->mk_ro_accessors(qw( index ));
31
32
=head1 NAME
33
34
Koha::ElasticSearch - Base module for things using elasticsearch
35
36
=head1 ACCESSORS
37
38
=over 4
39
40
=item index
41
42
The name of the index to use, generally 'biblios' or 'authorities'.
43
44
=back
45
46
=head1 FUNCTIONS
47
48
=cut
49
50
sub new {
51
    my $class = shift @_;
52
    my $self = $class->SUPER::new(@_);
53
    # Check for a valid index
54
    croak('No index name provided') unless $self->index;
55
    return $self;
56
}
57
58
=head2 get_elasticsearch_params
59
60
    my $params = $self->get_elasticsearch_params();
61
62
This provides a hashref that contains the parameters for connecting to the
63
ElasicSearch servers, in the form:
64
65
    {
66
        'servers' => ['127.0.0.1:9200', 'anotherserver:9200'],
67
        'index_name' => 'koha_instance',
68
    }
69
70
This is configured by the following in the C<config> block in koha-conf.xml:
71
72
    <elasticsearch>
73
        <server>127.0.0.1:9200</server>
74
        <server>anotherserver:9200</server>
75
        <index_name>koha_instance</index_name>
76
    </elasticsearch>
77
78
=cut
79
80
sub get_elasticsearch_params {
81
    my ($self) = @_;
82
83
    # Copy the hash so that we're not modifying the original
84
    my $es = { %{ C4::Context->config('elasticsearch') } };
85
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$es );
86
87
    # Helpfully, the multiple server lines end up in an array for us anyway
88
    # if there are multiple ones, but not if there's only one.
89
    my $server = $es->{server};
90
    delete $es->{server};
91
    if ( ref($server) eq 'ARRAY' ) {
92
93
        # store it called 'servers'
94
        $es->{servers} = $server;
95
    }
96
    elsif ($server) {
97
        $es->{servers} = [$server];
98
    }
99
    else {
100
        die "No elasticsearch servers were specified in koha-conf.xml.\n";
101
    }
102
    die "No elasticserver index_name was specified in koha-conf.xml.\n"
103
      if ( !$es->{index_name} );
104
    # Append the name of this particular index to our namespace
105
    $es->{index_name} .= '_' . $self->index;
106
    return $es;
107
}
108
109
=head2 get_elasticsearch_settings
110
111
    my $settings = $self->get_elasticsearch_settings();
112
113
This provides the settings provided to elasticsearch when an index is created.
114
These can do things like define tokenisation methods.
115
116
A hashref containing the settings is returned.
117
118
=cut
119
120
sub get_elasticsearch_settings {
121
    my ($self) = @_;
122
123
    # Ultimately this should come from a file or something, and not be
124
    # hardcoded.
125
    my $settings = {
126
        index => {
127
            analysis => {
128
                analyzer => {
129
                    analyser_phrase => {
130
                        tokenizer => 'keyword',
131
                        filter    => 'lowercase',
132
                    },
133
                    analyser_standard => {
134
                        tokenizer => 'standard',
135
                        filter    => 'lowercase',
136
                    }
137
                }
138
            }
139
        }
140
    };
141
    return $settings;
142
}
143
144
=head2 get_elasticsearch_mappings
145
146
    my $mappings = $self->get_elasticsearch_mappings();
147
148
This provides the mappings that get passed to elasticsearch when an index is
149
created.
150
151
=cut
152
153
sub get_elasticsearch_mappings {
154
    my ($self) = @_;
155
156
    my $mappings = {
157
        data => {
158
            properties => {
159
                record => {
160
                    store          => "yes",
161
                    include_in_all => "false",
162
                    type           => "string",
163
                },
164
            }
165
        }
166
    };
167
    $self->_foreach_mapping(
168
        sub {
169
            my ( undef, $name, $type, $facet ) = @_;
170
171
            # TODO if this gets any sort of complexity to it, it should
172
            # be broken out into its own function.
173
174
            # TODO be aware of date formats, but this requires pre-parsing
175
            # as ES will simply reject anything with an invalid date.
176
            my $es_type =
177
              $type eq 'boolean'
178
              ? 'boolean'
179
              : 'string';
180
            $mappings->{data}{properties}{$name} = {
181
                search_analyzer => "analyser_standard",
182
                index_analyzer  => "analyser_standard",
183
                type            => $es_type,
184
                fields          => {
185
                    phrase => {
186
                        search_analyzer => "analyser_phrase",
187
                        index_analyzer  => "analyser_phrase",
188
                        type            => "string"
189
                    },
190
                },
191
            };
192
            $mappings->{data}{properties}{$name}{null_value} = 0
193
              if $type eq 'boolean';
194
            if ($facet) {
195
                $mappings->{data}{properties}{ $name . '__facet' } = {
196
                    type  => "string",
197
                    index => "not_analyzed",
198
                };
199
            }
200
        }
201
    );
202
    return $mappings;
203
}
204
205
# Provides the rules for data conversion.
206
sub get_fixer_rules {
207
    my ($self) = @_;
208
209
    my $marcflavour = lc C4::Context->preference('marcflavour');
210
    my @rules;
211
    $self->_foreach_mapping(
212
        sub {
213
            my ( undef, $name, $type, $facet, $marcs ) = @_;
214
            my $field = $marcs->{$marcflavour};
215
            return unless defined $marcs->{$marcflavour};
216
            my $options = '';
217
218
            # There's a bug when using 'split' with something that
219
            # selects a range
220
            # The split makes everything into nested arrays, but that's not
221
            # really a big deal, ES doesn't mind.
222
            $options = '-split => 1' unless $field =~ m|_/| || $type eq 'sum';
223
            push @rules, "marc_map('$field','${name}', $options)";
224
            if ($facet) {
225
                push @rules, "marc_map('$field','${name}__facet', $options)";
226
            }
227
            if ( $type eq 'boolean' ) {
228
229
                # boolean gets special handling, basically if it doesn't exist,
230
                # it's added and set to false. Otherwise we can't query it.
231
                push @rules,
232
                  "unless exists('$name') add_field('$name', 0) end";
233
            }
234
            if ($type eq 'sum' ) {
235
                push @rules, "sum('$name')";
236
            }
237
        }
238
    );
239
240
    return \@rules;
241
}
242
243
=head2 _foreach_mapping
244
245
    $self->_foreach_mapping(
246
        sub {
247
            my ( $id, $name, $type, $facet, $marcs ) = @_;
248
            my $marc = $marcs->{marc21};
249
        }
250
    );
251
252
This allows you to apply a function to each entry in the elasticsearch mappings
253
table, in order to build the mappings for whatever is needed.
254
255
In the provided function, the files are:
256
257
=over 4
258
259
=item C<$id>
260
261
An ID number, corresponding to the entry in the database.
262
263
=item C<$name>
264
265
The field name for elasticsearch (corresponds to the 'mapping' column in the
266
database.
267
268
=item C<$type>
269
270
The type for this value, e.g. 'string'.
271
272
=item C<$facet>
273
274
True if this value should be facetised. This only really makes sense if the
275
field is understood by the facet processing code anyway.
276
277
=item C<$marc>
278
279
A hashref containing the MARC field specifiers for each MARC type. It's quite
280
possible for this to be undefined if there is otherwise an entry in a
281
different MARC form.
282
283
=back
284
285
=cut
286
287
sub _foreach_mapping {
288
    my ( $self, $sub ) = @_;
289
290
    # TODO use a caching framework here
291
    my $database = Koha::Database->new();
292
    my $schema   = $database->schema();
293
    my $rs       = $schema->resultset('ElasticsearchMapping')->search();
294
    for my $row ( $rs->all ) {
295
        $sub->(
296
            $row->id,
297
            $row->mapping,
298
            $row->type,
299
            $row->facet,
300
            {
301
                marc21  => $row->marc21,
302
                unimarc => $row->unimarc,
303
                normarc => $row->normarc
304
            }
305
        );
306
    }
307
}
308
309
1;
310
311
__END__
312
313
=head1 AUTHOR
314
315
=over 4
316
317
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
318
319
=item Robin Sheat C<< <robin@catalyst.net.nz> >>
320
321
=back
322
323
=cut
324
(-)a/Koha/ElasticSearch/Indexer.pm (+156 lines)
Line 0 Link Here
1
package Koha::ElasticSearch::Indexer;
2
3
# Copyright 2013 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Carp;
21
use Modern::Perl;
22
use base qw(Koha::ElasticSearch);
23
use Data::Dumper;
24
25
# For now just marc, but we can do anything here really
26
use Catmandu::Importer::MARC;
27
use Catmandu::Store::ElasticSearch;
28
29
Koha::ElasticSearch::Indexer->mk_accessors(qw( store ));
30
31
=head1 NAME
32
33
Koha::ElasticSearch::Indexer - handles adding new records to the index
34
35
=head1 SYNOPSIS
36
37
    my $indexer = Koha::ElasticSearch::Indexer->new({ index => 'biblios' });
38
    $indexer->delete_index();
39
    $indexer->update_index(\@biblionumbers, \@records);
40
41
=head1 FUNCTIONS
42
43
=cut
44
45
=head2 $indexer->update_index($biblionums, $records);
46
47
C<$biblionums> is an arrayref containing the biblionumbers for the records.
48
49
C<$records> is an arrayref containing the L<MARC::Record>s themselves.
50
51
The values in the arrays must match up, and the 999$c value in the MARC record
52
will be rewritten using the values in C<$biblionums> to ensure they are correct.
53
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
54
999$c is correct on your own then.
55
56
Note that this will modify the original record if C<$biblionums> is supplied.
57
If that's a problem, clone them first.
58
59
=cut
60
61
sub update_index {
62
    my ($self, $biblionums, $records) = @_;
63
64
    if ($biblionums) {
65
        $self->_sanitise_records($biblionums, $records);
66
    }
67
68
    my $from    = $self->_convert_marc_to_json($records);
69
    if ( !$self->store ) {
70
        my $params  = $self->get_elasticsearch_params();
71
        $self->store(
72
            Catmandu::Store::ElasticSearch->new(
73
                %$params,
74
                index_settings => $self->get_elasticsearch_settings(),
75
                index_mappings => $self->get_elasticsearch_mappings(),
76
                #trace_calls => 1,
77
            )
78
        );
79
    }
80
    $self->store->bag->add_many($from);
81
    $self->store->bag->commit;
82
    return 1;
83
}
84
85
=head2 $indexer->delete_index();
86
87
Deletes the index from the elasticsearch server. Calling C<update_index>
88
after this will recreate it again.
89
90
=cut
91
92
sub delete_index {
93
    my ($self) = @_;
94
95
    if (!$self->store) {
96
        # If this index doesn't exist, this will create it. Then it'll be
97
        # deleted. That's not the end of the world however.
98
        my $params  = $self->get_elasticsearch_params();
99
        $self->store(
100
            Catmandu::Store::ElasticSearch->new(
101
                %$params,
102
                index_settings => $self->get_elasticsearch_settings(),
103
                index_mappings => $self->get_elasticsearch_mappings(),
104
                #trace_calls => 1,
105
            )
106
        );
107
    }
108
    $self->store->drop();
109
    $self->store(undef);
110
}
111
112
sub _sanitise_records {
113
    my ($self, $biblionums, $records) = @_;
114
115
    confess "Unequal number of values in \$biblionums and \$records." if (@$biblionums != @$records);
116
117
    my $c = @$biblionums;
118
    for (my $i=0; $i<$c; $i++) {
119
        my $bibnum = $biblionums->[$i];
120
        my $rec = $records->[$i];
121
        # I've seen things you people wouldn't believe. Attack ships on fire
122
        # off the shoulder of Orion. I watched C-beams glitter in the dark near
123
        # the Tannhauser gate. MARC records where 999$c doesn't match the
124
        # biblionumber column. All those moments will be lost in time... like
125
        # tears in rain...
126
        $rec->delete_fields($rec->field('999'));
127
        $rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));
128
    }
129
}
130
131
sub _convert_marc_to_json {
132
    my $self    = shift;
133
    my $records = shift;
134
    my $importer =
135
      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
136
    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
137
    $importer = $fixer->fix($importer);
138
    return $importer;
139
}
140
141
1;
142
143
__END__
144
145
=head1 AUTHOR
146
147
=over 4
148
149
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
150
151
=item Robin Sheat C<< <robin@catalyst.net.nz> >>
152
153
=back
154
155
=cut
156
(-)a/Koha/ElasticSearch/Search.pm (+231 lines)
Line 0 Link Here
1
package Koha::ElasticSearch::Search;
2
3
# Copyright 2014 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
=head1 NAME
21
22
Koha::ElasticSearch::Search - search functions for Elasticsearch
23
24
=head1 SYNOPSIS
25
26
    my $searcher = Koha::ElasticSearch::Search->new();
27
    my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
28
    my $query = $builder->build_query('perl');
29
    my $results = $searcher->search($query);
30
    print "There were " . $results->total . " results.\n";
31
    $results->each(sub {
32
        push @hits, @_[0];
33
    });
34
35
=head1 METHODS
36
37
=cut
38
39
use base qw(Koha::ElasticSearch);
40
use Koha::ItemTypes;
41
42
use Catmandu::Store::ElasticSearch;
43
44
use Data::Dumper; #TODO remove
45
use Carp qw(cluck);
46
47
Koha::ElasticSearch::Search->mk_accessors(qw( store ));
48
49
=head2 search
50
51
    my $results = $searcher->search($query, $page, $count);
52
53
Run a search using the query. It'll return C<$count> results, starting at page
54
C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
55
56
C<%options> is a hash containing extra options:
57
58
=over 4
59
60
=item offset
61
62
If provided, this overrides the C<$page> value, and specifies the record as
63
an offset (i.e. the number of the record to start with), rather than a page.
64
65
=back
66
67
=cut
68
69
sub search {
70
    my ($self, $query, $page, $count, %options) = @_;
71
72
    my $params = $self->get_elasticsearch_params();
73
    my %paging;
74
    $paging{limit} = $count || 20;
75
    # ES doesn't want pages, it wants a record to start from.
76
    if (exists $options{offset}) {
77
        $paging{start} = $options{offset};
78
    } else {
79
        $page = (!defined($page) || ($page <= 0)) ? 1 : $page - 1;
80
        $paging{start} = $page * $paging{limit};
81
    }
82
    $self->store(
83
        Catmandu::Store::ElasticSearch->new(
84
            %$params,
85
            trace_calls => 0,
86
        )
87
    );
88
    my $results = $self->store->bag->search( %$query, %paging );
89
    return $results;
90
}
91
92
=head2 search_compat
93
94
    my ( $error, $results, $facets ) = $search->search_compat(
95
        $query,            $simple_query, \@sort_by,       \@servers,
96
        $results_per_page, $offset,       $expanded_facet, $branches,
97
        $query_type,       $scan
98
      )
99
100
A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
101
that is returned in the query created by build_query_compat will probably
102
get ignored here.
103
104
=cut
105
106
sub search_compat {
107
    my (
108
        $self,     $query,            $simple_query, $sort_by,
109
        $servers,  $results_per_page, $offset,       $expanded_facet,
110
        $branches, $query_type,       $scan
111
    ) = @_;
112
113
    my %options;
114
    $options{offset} = $offset;
115
    my $results = $self->search($query, undef, $results_per_page, %options);
116
117
    # Convert each result into a MARC::Record
118
    my (@records, $index);
119
    $index = $offset; # opac-search expects results to be put in the
120
        # right place in the array, according to $offset
121
    $results->each(sub {
122
            # The results come in an array for some reason
123
            my $marc_json = @_[0]->{record};
124
            my $marc = $self->json2marc($marc_json);
125
            $records[$index++] = $marc;
126
        });
127
    # consumers of this expect a name-spaced result, we provide the default
128
    # configuration.
129
    my %result;
130
    $result{biblioserver}{hits} = $results->total;
131
    $result{biblioserver}{RECORDS} = \@records;
132
    return (undef, \%result, $self->_convert_facets($results->{facets}));
133
}
134
135
=head2 json2marc
136
137
    my $marc = $self->json2marc($marc_json);
138
139
Converts the form of marc (based on its JSON, but as a Perl structure) that
140
Catmandu stores into a MARC::Record object.
141
142
=cut
143
144
sub json2marc {
145
    my ( $self, $marcjson ) = @_;
146
147
    my $marc = MARC::Record->new();
148
    $marc->encoding('UTF-8');
149
150
    # fields are like:
151
    # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
152
    # conveniently, this is the form that MARC::Field->new() likes
153
    foreach $field (@$marcjson) {
154
        next if @$field < 5;    # Shouldn't be possible, but...
155
        if ( $field->[0] eq 'LDR' ) {
156
            $marc->leader( $field->[4] );
157
        }
158
        else {
159
            my $marc_field = MARC::Field->new(@$field);
160
            $marc->append_fields($marc_field);
161
        }
162
    }
163
    return $marc;
164
}
165
166
=head2 _convert_facets
167
168
    my $koha_facets = _convert_facets($es_facets);
169
170
Converts elasticsearch facets types to the form that Koha expects.
171
It expects the ES facet name to match the Koha type, for example C<itype>,
172
C<au>, C<su-to>, etc.
173
174
=cut
175
176
sub _convert_facets {
177
    my ( $self, $es ) = @_;
178
179
    return undef if !$es;
180
181
    # These should correspond to the ES field names, as opposed to the CCL
182
    # things that zebra uses.
183
    my %type_to_label = (
184
        author   => 'Authors',
185
        location => 'Location',
186
        itype    => 'ItemTypes',
187
        se       => 'Series',
188
        subject  => 'Topics',
189
        'su-geo' => 'Places',
190
    );
191
192
    # We also have some special cases, e.g. itypes that need to show the
193
    # value rather than the code.
194
    my $itypes = Koha::ItemTypes->new();
195
    my %special = ( itype => sub { $itypes->get_description_for_code(@_) }, );
196
    my @res;
197
    while ( ( $type, $data ) = each %$es ) {
198
        next if !exists( $type_to_label{$type} );
199
        my $facet = {
200
            type_id => $type . '_id',
201
            expand  => $type,
202
            expandable => 1,    # TODO figure how that's supposed to work
203
            "type_label_$type_to_label{$type}" => 1,
204
            type_link_value                    => $type,
205
        };
206
        foreach my $term ( @{ $data->{terms} } ) {
207
            my $t = $term->{term};
208
            my $c = $term->{count};
209
            if ( exists( $special{$type} ) ) {
210
                $label = $special{$type}->($t);
211
            }
212
            else {
213
                $label = $t;
214
            }
215
            push @{ $facet->{facets} }, {
216
                facet_count       => $c,
217
                facet_link_value  => $t,
218
                facet_title_value => $t . " ($c)",
219
                facet_label_value => $label,    # TODO either truncate this,
220
                     # or make the template do it like it should anyway
221
                type_link_value => $type,
222
            };
223
        }
224
        push @res, $facet if exists $facet->{facets};
225
    }
226
    return \@res;
227
}
228
229
230
1;
231
(-)a/Koha/ItemType.pm (+70 lines)
Line 0 Link Here
1
package Koha::ItemType;
2
3
# This represents a single itemtype
4
5
# Copyright 2014 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
=head1 NAME
23
24
Koha::ItemType - represents a single itemtype
25
26
=head1 DESCRIPTION
27
28
This contains the data relating to a single itemtype
29
30
=head1 SYNOPSIS
31
32
    use Koha::ItemTypes;
33
    my $types = Koha::ItemTypes->new();
34
    my $type  = $types->get_itemtype('CODE');
35
    print $type->code, $type->description, $type->rentalcharge,
36
      $type->imageurl, $type->summary, $type->checkinmsg,
37
      $type->checkinmsgtype;
38
39
Creating an instance of C<Koha::ItemType> without using L<Koha::ItemTypes>
40
can be done simply by passing a hashref containing the values to C<new()>.
41
Note when doing this that a value for C<itemtype> will become a value for
42
C<code>.
43
44
=head1 FUNCTIONS
45
46
In addition to the read-only accessors mentioned above, the following functions
47
exist.
48
49
=cut
50
51
use Modern::Perl;
52
53
use base qw(Class::Accessor);
54
55
# TODO can we make these auto-generate from the input hash so it doesn't
56
# have to be updated when the database is?
57
__PACKAGE__->mk_ro_accessors(
58
    qw(code description rentalcharge imageurl
59
      summary checkinmsg checkinmsgtype)
60
);
61
62
sub new {
63
    my $class = shift @_;
64
65
    my %data = ( %{ $_[0] }, code => $_[0]->{itemtype} );
66
    my $self = $class->SUPER::new( \%data );
67
    return $self;
68
}
69
70
1;
(-)a/Koha/ItemTypes.pm (+113 lines)
Line 0 Link Here
1
package Koha::ItemTypes;
2
3
# This contains the item types that the system knows about.
4
5
# Copyright 2014 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
=head1 NAME
23
24
Koha::ItemTypes - handles the item types that Koha knows about
25
26
=head1 DESCRIPTION
27
28
This contains functions to access the item types.
29
30
Note that any changes that happen to the database while this object is live
31
may not be reflected, so best don't hold onto it for a long time
32
33
=cut
34
35
use Koha::Database;
36
use Koha::ItemType;
37
use Modern::Perl;
38
39
use Data::Dumper; # TODO remove
40
use base qw(Class::Accessor);
41
42
__PACKAGE__->mk_accessors(qw());
43
44
=head1 FUNCTIONS
45
46
=head2 new
47
48
    my $itypes = Koha::ItemTypes->new();
49
50
Creates a new instance of the object.
51
52
=cut
53
54
# Handled by Class::Accessor
55
56
=head2 get_itemtype
57
58
    my @itype = $itypes->get_itemtype('CODE1', 'CODE2');
59
60
This returns a L<Koha::ItemType> object for each of the provided codes. For
61
any that don't exist, an C<undef> is returned.
62
63
=cut
64
65
sub get_itemtype {
66
    my ($self, @codes) = @_;
67
68
    my $schema = Koha::Database->new()->schema();
69
    my @res;
70
71
    foreach my $c (@codes) {
72
        if (exists $self->{cached}{$c}) {
73
            push @res, $self->{cached}{$c};
74
            next;
75
        }
76
        my $rs = $schema->resultset('Itemtype')->search( { itemtype => $c } );
77
        my $r = $rs->next;
78
        if (!$r) {
79
            push @res, undef;
80
            next;
81
        }
82
        my %data = $r->get_inflated_columns;
83
        my $it = Koha::ItemType->new(\%data);
84
        push @res, $it;
85
        $self->{cached}{$c} = $it;
86
    }
87
    if (wantarray) {
88
        return @res;
89
    } else {
90
        return @res ? $res[0] : undef;
91
    }
92
}
93
94
=head2 get_description_for_code
95
96
    my $desc = $itypes->get_description_for_code($code);
97
98
This returns the description for an itemtype code. As a special case, if
99
there is no itemtype for this code, it'll return what it was given.
100
101
It is mostly as a convenience function rather than using L<get_itemtype>.
102
103
=cut
104
105
sub get_description_for_code {
106
    my ($self, $code) = @_;
107
108
    my $itype = $self->get_itemtype($code);
109
    return $code if !$itype;
110
    return $itype->description;
111
}
112
113
1;
(-)a/Koha/Schema/Result/ElasticsearchMapping.pm (+105 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::ElasticsearchMapping;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::ElasticsearchMapping
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<elasticsearch_mapping>
19
20
=cut
21
22
__PACKAGE__->table("elasticsearch_mapping");
23
24
=head1 ACCESSORS
25
26
=head2 id
27
28
  data_type: 'integer'
29
  is_auto_increment: 1
30
  is_nullable: 0
31
32
=head2 mapping
33
34
  data_type: 'varchar'
35
  is_nullable: 1
36
  size: 255
37
38
=head2 type
39
40
  data_type: 'varchar'
41
  is_nullable: 1
42
  size: 255
43
44
=head2 facet
45
46
  data_type: 'tinyint'
47
  default_value: 0
48
  is_nullable: 1
49
50
=head2 marc21
51
52
  data_type: 'varchar'
53
  is_nullable: 1
54
  size: 255
55
56
=head2 unimarc
57
58
  data_type: 'varchar'
59
  is_nullable: 1
60
  size: 255
61
62
=head2 normarc
63
64
  data_type: 'varchar'
65
  is_nullable: 1
66
  size: 255
67
68
=cut
69
70
__PACKAGE__->add_columns(
71
  "id",
72
  { data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
73
  "mapping",
74
  { data_type => "varchar", is_nullable => 1, size => 255 },
75
  "type",
76
  { data_type => "varchar", is_nullable => 1, size => 255 },
77
  "facet",
78
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
79
  "marc21",
80
  { data_type => "varchar", is_nullable => 1, size => 255 },
81
  "unimarc",
82
  { data_type => "varchar", is_nullable => 1, size => 255 },
83
  "normarc",
84
  { data_type => "varchar", is_nullable => 1, size => 255 },
85
);
86
87
=head1 PRIMARY KEY
88
89
=over 4
90
91
=item * L</id>
92
93
=back
94
95
=cut
96
97
__PACKAGE__->set_primary_key("id");
98
99
100
# Created by DBIx::Class::Schema::Loader v0.07040 @ 2014-06-06 16:20:16
101
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uGRmWU0rshP6awyLMQYJeQ
102
103
104
# You can replace this text with custom code or comments, and it will be preserved on regeneration
105
1;
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (+498 lines)
Line 0 Link Here
1
package Koha::SearchEngine::Elasticsearch::QueryBuilder;
2
3
# This file is part of Koha.
4
#
5
# Copyright 2014 Catalyst IT Ltd.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
=head1 NAME
21
22
Koha::SearchEngine::Elasticsearch::QueryBuilder - constructs elasticsearch
23
query objects from user-supplied queries
24
25
=head1 DESCRIPTION
26
27
This provides the functions that take a user-supplied search query, and
28
provides something that can be given to elasticsearch to get answers.
29
30
=head1 SYNOPSIS
31
32
    use Koha::SearchEngine::Elasticsearch;
33
    $builder = Koha::SearchEngine::Elasticsearch->new();
34
    my $simple_query = $builder->build_query("hello");
35
    # This is currently undocumented because the original code is undocumented
36
    my $adv_query = $builder->build_advanced_query($indexes, $operands, $operators);
37
38
=head1 METHODS
39
40
=cut
41
42
use base qw(Class::Accessor);
43
use List::MoreUtils qw/ each_array /;
44
use Modern::Perl;
45
use URI::Escape;
46
47
use Data::Dumper;    # TODO remove
48
49
=head2 build_query
50
51
    my $simple_query = $builder->build_query("hello", %options)
52
53
This will build a query that can be issued to elasticsearch from the provided
54
string input. This expects a lucene style search form (see
55
L<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax>
56
for details.)
57
58
It'll make an attempt to respect the various query options.
59
60
Additional options can be provided with the C<%options> hash.
61
62
=over 4
63
64
=item sort
65
66
This should be an arrayref of hashrefs, each containing a C<field> and an
67
C<direction> (optional, defaults to C<asc>.) The results will be sorted
68
according to these values. Valid values for C<direction> are 'asc' and 'desc'.
69
70
=back
71
72
=cut
73
74
sub build_query {
75
    my ( $self, $query, %options ) = @_;
76
77
    my $stemming         = C4::Context->preference("QueryStemming")        || 0;
78
    my $auto_truncation  = C4::Context->preference("QueryAutoTruncate")    || 0;
79
    my $weight_fields    = C4::Context->preference("QueryWeightFields")    || 0;
80
    my $fuzzy_enabled    = C4::Context->preference("QueryFuzzy")           || 0;
81
82
    $query = '*' unless defined $query;
83
84
    my $res;
85
    $res->{query} = {
86
        query_string => {
87
            query            => $query,
88
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
89
            default_operator => "AND",
90
            default_field    => "_all",
91
        }
92
    };
93
94
    if ( $options{sort} ) {
95
        foreach my $sort ( @{ $options{sort} } ) {
96
            my ( $f, $d ) = @$sort{qw/ field direction /};
97
            die "Invalid sort direction, $d"
98
              if $d && ( $d ne 'asc' && $d ne 'desc' );
99
            $d = 'asc' unless $d;
100
101
            # TODO account for fields that don't have a 'phrase' type
102
            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
103
        }
104
    }
105
106
    # See _convert_facets in Search.pm for how these get turned into
107
    # things that Koha can use.
108
    $res->{facets} = {
109
        author  => { terms => { field => "author__facet" } },
110
        subject => { terms => { field => "subject__facet" } },
111
        itype   => { terms => { field => "itype__facet" } },
112
    };
113
    return $res;
114
}
115
116
=head2 build_browse_query
117
118
    my $browse_query = $builder->build_browse_query($field, $query);
119
120
This performs a "starts with" style query on a particular field. The field
121
to be searched must have been indexed with an appropriate mapping as a
122
"phrase" subfield.
123
124
=cut
125
126
sub build_browse_query {
127
    my ( $self, $field, $query ) = @_;
128
129
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
130
131
    return { query => '*' } if !defined $query;
132
133
    # TODO this should come from Koha::Elasticsearch
134
    my %field_whitelist = (
135
        title  => 1,
136
        author => 1,
137
    );
138
    $field = 'title' if !exists $field_whitelist{$field};
139
140
    my $res = {
141
        query => {
142
            match_phrase_prefix => {
143
                "$field.phrase" => {
144
                    query     => $query,
145
                    operator  => 'or',
146
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
147
                }
148
            }
149
        },
150
        sort => [ { "$field.phrase" => { order => "asc" } } ],
151
    };
152
}
153
154
=head2 build_query_compat
155
156
    my (
157
        $error,             $query, $simple_query, $query_cgi,
158
        $query_desc,        $limit, $limit_cgi,    $limit_desc,
159
        $stopwords_removed, $query_type
160
      )
161
      = $builder->build_query_compat( \@operators, \@operands, \@indexes,
162
        \@limits, \@sort_by, $scan, $lang );
163
164
This handles a search using the same api as L<C4::Search::buildQuery> does.
165
166
A very simple query will go in with C<$operands> set to ['query'], and
167
C<$sort_by> set to ['pubdate_dsc']. This simple case will return with
168
C<$query> set to something that can perform the search, C<$simple_query>
169
set to just the search term, C<$query_cgi> set to something that can
170
reproduce this search, and C<$query_desc> set to something else.
171
172
=cut
173
174
sub build_query_compat {
175
    my ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan,
176
        $lang )
177
      = @_;
178
179
#die Dumper ( $self, $operators, $operands, $indexes, $limits, $sort_by, $scan, $lang );
180
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
181
    my @index_params = $self->_convert_index_fields(@$indexes);
182
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
183
184
    # Merge the indexes in with the search terms and the operands so that
185
    # each search thing is a handy unit.
186
    unshift @$operators, undef;    # The first one can't have an op
187
    my @search_params;
188
    my $ea = each_array( @$operands, @$operators, @index_params );
189
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
190
        next if ( !defined($oand) || $oand eq '' );
191
        push @search_params, {
192
            operand => $self->_clean_search_term($oand),      # the search terms
193
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
194
            $index ? %$index : (),
195
        };
196
    }
197
198
    # We build a string query from limits and the queries. An alternative
199
    # would be to pass them separately into build_query and let it build
200
    # them into a structured ES query itself. Maybe later, though that'd be
201
    # more robust.
202
    my $query_str = join( ' AND ',
203
        join( ' ', $self->_create_query_string(@search_params) ),
204
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) );
205
206
    # If there's no query on the left, let's remove the junk left behind
207
    $query_str =~ s/^ AND //;
208
    my %options;
209
    $options{sort} = \@sort_params;
210
    my $query = $self->build_query( $query_str, %options );
211
212
    #die Dumper($query);
213
    # We roughly emulate the CGI parameters of the zebra query builder
214
    my $query_cgi = 'idx=kw&q=' . uri_escape( $operands->[0] ) if @$operands;
215
    my $simple_query = $operands->[0] if @$operands == 1;
216
    my $query_desc   = $simple_query;
217
    my $limit        = 'and ' . join( ' and ', @$limits );
218
    my $limit_cgi =
219
      '&limit=' . join( '&limit=', map { uri_escape($_) } @$orig_limits );
220
    my $limit_desc = "@$limits";
221
222
    return (
223
        undef,  $query,     $simple_query, $query_cgi, $query_desc,
224
        $limit, $limit_cgi, $limit_desc,   undef,      undef
225
    );
226
}
227
228
=head2 _convert_sort_fields 
229
230
    my @sort_params = _convert_sort_fields(@sort_by)
231
232
Converts the zebra-style sort index information into elasticsearch-style.
233
234
C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
235
something that can be sent to L<build_query>.
236
237
=cut
238
239
sub _convert_sort_fields {
240
    my ( $self, @sort_by ) = @_;
241
242
    # Turn the sorting into something we care about.
243
    my %sort_field_convert = (
244
        acqdate     => 'acqdate',
245
        author      => 'author',
246
        call_number => 'callnum',
247
        popularity  => 'issues',
248
        relevance   => undef,       # default
249
        title       => 'title',
250
        pubdate     => 'pubdate',
251
    );
252
    my %sort_order_convert =
253
      ( qw( dsc desc ), qw( asc asc ), qw( az asc ), qw( za desc ) );
254
255
    # Convert the fields and orders, drop anything we don't know about.
256
    grep { $_->{field} } map {
257
        my ( $f, $d ) = split /_/;
258
        {
259
            field     => $sort_field_convert{$f},
260
            direction => $sort_order_convert{$d}
261
        }
262
    } @sort_by;
263
}
264
265
=head2 _convert_index_fields
266
267
    my @index_params = $self->_convert_index_fields(@indexes);
268
269
Converts zebra-style search index notation into elasticsearch-style.
270
271
C<@indexes> is an array of index names, as presented to L<build_query_compat>,
272
and it returns something that can be sent to L<build_query>.
273
274
B<TODO>: this will pull from the elasticsearch mappings table to figure out
275
types.
276
277
=cut
278
279
our %index_field_convert = (
280
    'kw'       => '_all',
281
    'ti'       => 'title',
282
    'au'       => 'author',
283
    'su'       => 'subject',
284
    'nb'       => 'isbn',
285
    'se'       => 'title-series',
286
    'callnum'  => 'callnum',
287
    'mc-itype' => 'itype',
288
    'ln'       => 'ln',
289
    'branch'   => 'homebranch',
290
    'fic'      => 'lf',
291
    'mus'      => 'rtype',
292
    'aud'      => 'ta',
293
);
294
295
sub _convert_index_fields {
296
    my ( $self, @indexes ) = @_;
297
298
    my %index_type_convert =
299
      ( __default => undef, phr => 'phrase', rtrn => 'right-truncate' );
300
301
    # Convert according to our table, drop anything that doesn't convert
302
    grep { $_->{field} } map {
303
        my ( $f, $t ) = split /,/;
304
        {
305
            field => $index_field_convert{$f},
306
            type  => $index_type_convert{ $t // '__default' }
307
        }
308
    } @indexes;
309
}
310
311
=head2 _convert_index_strings
312
313
    my @searches = $self->_convert_index_strings(@searches);
314
315
Similar to L<_convert_index_fields>, this takes strings of the form
316
B<field:search term> and rewrites the field from zebra-style to
317
elasticsearch-style. Anything it doesn't understand is returned verbatim.
318
319
=cut
320
321
sub _convert_index_strings {
322
    my ( $self, @searches ) = @_;
323
324
    my @res;
325
    foreach my $s (@searches) {
326
        next if $s eq '';
327
        my ( $field, $term ) = $s =~ /^\s*([\w,-]*?):(.*)/;
328
        unless ( defined($field) && defined($term) ) {
329
            push @res, $s;
330
            next;
331
        }
332
        my ($conv) = $self->_convert_index_fields($field);
333
        unless ( defined($conv) ) {
334
            push @res, $s;
335
            next;
336
        }
337
        push @res, $conv->{field} . ":"
338
          . $self->_modify_string_by_type( %$conv, operand => $term );
339
    }
340
    return @res;
341
}
342
343
=head2 _modify_string_by_type
344
345
    my $str = $self->_modify_string_by_type(%index_field);
346
347
If you have a search term (operand) and a type (phrase, right-truncated), this
348
will convert the string to have the function in lucene search terms, e.g.
349
wrapping quotes around it.
350
351
=cut
352
353
sub _modify_string_by_type {
354
    my ( $self, %idx ) = @_;
355
356
    my $type = $idx{type} || '';
357
    my $str = $idx{operand};
358
    return $str unless $str;    # Empty or undef, we can't use it.
359
360
    $str .= '*' if $type eq 'right-truncate';
361
    $str = '"' . $str . '"' if $type eq 'phrase';
362
    return $str;
363
}
364
365
=head2 _convert_index_strings_freeform
366
367
    my $search = $self->_convert_index_strings_freeform($search);
368
369
This is similar to L<_convert_index_strings>, however it'll search out the
370
things to change within the string. So it can handle strings such as
371
C<(su:foo) AND (su:bar)>, converting the C<su> appropriately.
372
373
=cut
374
375
sub _convert_index_strings_freeform {
376
    my ( $self, $search ) = @_;
377
378
    while ( my ( $zeb, $es ) = each %index_field_convert ) {
379
        $search =~ s/\b$zeb:/$es:/g;
380
    }
381
    return $search;
382
}
383
384
=head2 _join_queries
385
386
    my $query_str = $self->_join_queries(@query_parts);
387
388
This takes a list of query parts, that might be search terms on their own, or
389
booleaned together, or specifying fields, or whatever, wraps them in
390
parentheses, and ANDs them all together. Suitable for feeding to the ES
391
query string query.
392
393
=cut
394
395
sub _join_queries {
396
    my ( $self, @parts ) = @_;
397
398
    @parts = grep { defined($_) && $_ ne '' } @parts;
399
    return () unless @parts;
400
    return $parts[0] if @parts < 2;
401
    join ' AND ', map { "($_)" } @parts;
402
}
403
404
=head2 _make_phrases
405
406
    my @phrased_queries = $self->_make_phrases(@query_parts);
407
408
This takes the supplied queries and forces them to be phrases by wrapping
409
quotes around them. It understands field prefixes, e.g. 'subject:' and puts
410
the quotes outside of them if they're there.
411
412
=cut
413
414
sub _make_phrases {
415
    my ( $self, @parts ) = @_;
416
    map { s/^\s*(\w*?:)(.*)$/$1"$2"/r } @parts;
417
}
418
419
=head2 _create_query_string
420
421
    my @query_strings = $self->_create_query_string(@queries);
422
423
Given a list of hashrefs, it will turn them into a lucene-style query string.
424
The hash should contain field, type (both for the indexes), operator, and
425
operand.
426
427
=cut
428
429
sub _create_query_string {
430
    my ( $self, @queries ) = @_;
431
432
    map {
433
        my $otor  = $_->{operator} ? $_->{operator} . ' ' : '';
434
        my $field = $_->{field}    ? $_->{field} . ':'    : '';
435
436
        my $oand = $self->_modify_string_by_type(%$_);
437
        "$otor($field$oand)";
438
    } @queries;
439
}
440
441
=head2 _clean_search_term
442
443
    my $term = $self->_clean_search_term($term);
444
445
This cleans a search term by removing any funny characters that may upset
446
ES and give us an error. It also calls L<_convert_index_strings_freeform>
447
to ensure those parts are correct.
448
449
=cut
450
451
sub _clean_search_term {
452
    my ( $self, $term ) = @_;
453
454
    $term = $self->_convert_index_strings_freeform($term);
455
    $term =~ s/[{}]/"/g;
456
    return $term;
457
}
458
459
=head2 _fix_limit_special_cases
460
461
    my $limits = $self->_fix_limit_special_cases($limits);
462
463
This converts any special cases that the limit specifications have into things
464
that are more readily processable by the rest of the code.
465
466
The argument should be an arrayref, and it'll return an arrayref.
467
468
=cut
469
470
sub _fix_limit_special_cases {
471
    my ( $self, $limits ) = @_;
472
473
    my @new_lim;
474
    foreach my $l (@$limits) {
475
476
        # This is set up by opac-search.pl
477
        if ( $l =~ /^yr,st-numeric,ge=/ ) {
478
            my ( $start, $end ) =
479
              ( $l =~ /^yr,st-numeric,ge=(.*) and yr,st-numeric,le=(.*)$/ );
480
            next unless defined($start) && defined($end);
481
            push @new_lim, "copydate:[$start TO $end]";
482
        }
483
        elsif ( $l =~ /^yr,st-numeric=/ ) {
484
            my ($date) = ( $l =~ /^yr,st-numeric=(.*)$/ );
485
            next unless defined($date);
486
            push @new_lim, "copydate:$date";
487
        }
488
        elsif ( $l =~ /^available$/ ) {
489
            push @new_lim, 'onloan:false';
490
        }
491
        else {
492
            push @new_lim, $l;
493
        }
494
    }
495
    return \@new_lim;
496
}
497
498
1;
(-)a/Koha/SearchEngine/QueryBuilderRole.pm (+3 lines)
Lines 20-24 package Koha::SearchEngine::QueryBuilderRole; Link Here
20
use Moose::Role;
20
use Moose::Role;
21
21
22
requires 'build_query';
22
requires 'build_query';
23
# The compat version should accept and return parameters in the same form as
24
# C4::Search->buildQuery does.
25
requires 'build_query_compat';
23
26
24
1;
27
1;
(-)a/Koha/SearchEngine/Zebra.pm (-1 / +2 lines)
Lines 19-25 package Koha::SearchEngine::Zebra; Link Here
19
19
20
use Moose;
20
use Moose;
21
21
22
extends 'Data::SearchEngine::Zebra';
22
# Removed because it doesn't exist.
23
#extends 'Data::SearchEngine::Zebra';
23
24
24
# the configuration file is retrieved from KOHA_CONF by default, provide it from there²
25
# the configuration file is retrieved from KOHA_CONF by default, provide it from there²
25
has '+conf_file' => (
26
has '+conf_file' => (
(-)a/Koha/SearchEngine/Zebra/QueryBuilder.pm (+7 lines)
Lines 17-22 package Koha::SearchEngine::Zebra::QueryBuilder; Link Here
17
# You should have received a copy of the GNU General Public License
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
19
20
use base qw(Class::Accessor);
20
use Modern::Perl;
21
use Modern::Perl;
21
use Moose::Role;
22
use Moose::Role;
22
use C4::Search;
23
use C4::Search;
Lines 28-31 sub build_query { Link Here
28
    C4::Search::buildQuery @_;
29
    C4::Search::buildQuery @_;
29
}
30
}
30
31
32
sub build_query_compat {
33
    # Because this passes directly on to C4::Search, we have no trouble being
34
    # compatible.
35
    build_query(@_);
36
}
37
31
1;
38
1;
(-)a/Koha/SearchEngine/Zebra/Search.pm (-12 / +30 lines)
Lines 17-36 package Koha::SearchEngine::Zebra::Search; Link Here
17
# You should have received a copy of the GNU General Public License
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
19
20
use Moose::Role;
20
# I don't think this ever worked right
21
with 'Koha::SearchEngine::SearchRole';
21
#use Moose::Role;
22
#with 'Koha::SearchEngine::SearchRole';
22
23
23
use Data::SearchEngine::Zebra;
24
use base qw(Class::Accessor);
24
use Data::SearchEngine::Query;
25
# Removed because it doesn't exist/doesn't work.
25
use Koha::SearchEngine::Zebra;
26
#use Data::SearchEngine::Zebra;
26
use Data::Dump qw(dump);
27
#use Data::SearchEngine::Query;
28
#use Koha::SearchEngine::Zebra;
29
#use Data::Dump qw(dump);
27
30
28
has searchengine => (
31
use C4::Search; # :(
29
    is => 'rw',
32
30
    isa => 'Koha::SearchEngine::Zebra',
33
# Broken without the Data:: stuff
31
    default => sub { Koha::SearchEngine::Zebra->new },
34
#has searchengine => (
32
    lazy => 1
35
#    is => 'rw',
33
);
36
#    isa => 'Koha::SearchEngine::Zebra',
37
#    default => sub { Koha::SearchEngine::Zebra->new },
38
#    lazy => 1
39
#);
34
40
35
sub search {
41
sub search {
36
    my ($self,$query_string) = @_;
42
    my ($self,$query_string) = @_;
Lines 53-58 sub search { Link Here
53
    }
59
    }
54
}
60
}
55
61
62
=head2 search_compat
63
64
This passes straight through to C4::Search::getRecords.
65
66
=cut
67
68
sub search_compat {
69
    shift; # get rid of $self
70
71
    return getRecords(@_);
72
}
73
56
sub dosmth {'bou' }
74
sub dosmth {'bou' }
57
75
58
1;
76
1;
(-)a/installer/data/mysql/elasticsearch_mapping.sql (+149 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS elasticsearch_mapping;
2
CREATE TABLE `elasticsearch_mapping` (
3
  `id` int(11) NOT NULL AUTO_INCREMENT,
4
  `mapping` varchar(255) DEFAULT NULL,
5
  `type` varchar(255) NOT NULL,
6
  `facet` boolean DEFAULT FALSE,
7
  `marc21` varchar(255) DEFAULT NULL,
8
  `unimarc` varchar(255) DEFAULT NULL,
9
  `normarc` varchar(255) DEFAULT NULL,
10
  PRIMARY KEY (`id`)
11
) ENGINE=InnoDB AUTO_INCREMENT=126 DEFAULT CHARSET=utf8;
12
13
14
15
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('llength',FALSE,'','leader_/1-5',NULL,'leader_/1-5');
16
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('rtype',FALSE,'','leader_/6',NULL,'leader_/6');
17
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bib-level',FALSE,'','leader_/7',NULL,'leader_/7');
18
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('control-number',FALSE,'','001',NULL,'001');
19
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'',NULL,'001',NULL);
20
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('date-time-last-modified',FALSE,'','005','099d',NULL);
21
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('microform-generation',FALSE,'','007_/11',NULL,'007_/11');
22
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('material-type',FALSE,'','007','200b','007');
23
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-00',FALSE,'','007_/1',NULL,'007_/1');
24
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-01',FALSE,'','007_/2',NULL,'007_/2');
25
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-02',FALSE,'','007_/3',NULL,'007_/3');
26
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-01-02',FALSE,'','007_/1-2',NULL,'007_/1-2');
27
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('date-entered-on-file',FALSE,'','008_/1-5','099c','008_/1-5');
28
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('pubdate',FALSE,'','008_/7-10','100a_/9-12','008_/7-10');
29
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('pl',FALSE,'','008_/15-17','210a','008_/15-17');
30
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ta',FALSE,'','008_/22','100a_/17','008_/22');
31
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff8-23',FALSE,'','008_/23',NULL,'008_/23');
32
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff8-29',FALSE,'','008_/29','105a_/8','008_/29');
33
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lf',FALSE,'','008_/33','105a_/11','008_/33');
34
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bio',FALSE,'','008_/34','105a_/12','008_/34');
35
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ln',FALSE,'','008_/35-37','101a','008_/35-37');
36
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ctype',FALSE,'','008_/24-27','105a_/4-7','008_/24-27');
37
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('record-source',FALSE,'','008_/39','995c','008_/39');
38
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lc-cardnumber',FALSE,'','010','995j','010');
39
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lc-cardnumber',FALSE,'','011',NULL,NULL);
40
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','010',NULL,'010');
41
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','011',NULL,NULL);
42
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bnb-card-number',FALSE,'','015',NULL,'015');
43
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bgf-number',FALSE,'','015',NULL,'015');
44
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-db',FALSE,'','015',NULL,'015');
45
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-natl-biblio',FALSE,'','015',NULL,'015');
46
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','015',NULL,'015');
47
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-legal-deposit',FALSE,'','017',NULL,NULL);
48
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','017',NULL,NULL);
49
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','018',NULL,NULL);
50
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','020a','010az','020a');
51
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('isbn',FALSE,'','020a','010az','020a');
52
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','022a','011ayz','022a');
53
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('issn',FALSE,'','022a','011ayz','022a');
54
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','100a','200f','100a');
55
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','110a','200g','110a');
56
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','111a',NULL,'111a');
57
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','700a','700a','700a');
58
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',FALSE,'string','245c','701','245c');
59
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','245a','200a','245a');
60
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','246','200c','246');
61
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','247','200d','247');
62
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','490','200e','490a');
63
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','505t','200h',NULL);
64
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','711t','200i','711t');
65
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','700t','205','700t');
66
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','710t','304a','710t');
67
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','730','327a','730');
68
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','740','327b','740');
69
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','780','327c','780');
70
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','785','327d','785');
71
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','130','327e','130');
72
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','210','327f','210');
73
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','211','327g',NULL);
74
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','212','327h',NULL);
75
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','214','327i',NULL);
76
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','222','328t','222');
77
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','240','410t','240');
78
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'411t',NULL);
79
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'412t',NULL);
80
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'413t',NULL);
81
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'421t',NULL);
82
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'422t',NULL);
83
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'423t',NULL);
84
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'424t',NULL);
85
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'425t',NULL);
86
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'430t',NULL);
87
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'431t',NULL);
88
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'432t',NULL);
89
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'433t',NULL);
90
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'434t',NULL);
91
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'435t',NULL);
92
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'436t',NULL);
93
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'437t',NULL);
94
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'440t',NULL);
95
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'441t',NULL);
96
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'442t',NULL);
97
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'443t',NULL);
98
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'444t',NULL);
99
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'445t',NULL);
100
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'446t',NULL);
101
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'447t',NULL);
102
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'448t',NULL);
103
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'451t',NULL);
104
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'452t',NULL);
105
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'453t',NULL);
106
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'454t',NULL);
107
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'455t',NULL);
108
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'456t',NULL);
109
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'461t',NULL);
110
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'462t',NULL);
111
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'463t',NULL);
112
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'464t',NULL);
113
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'470t',NULL);
114
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'481t',NULL);
115
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'482t',NULL);
116
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'488t',NULL);
117
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','600a','600a','600a');
118
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','600t','600','600t');
119
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','610a','601','610a');
120
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','610t','602','610t');
121
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','611','604','611');
122
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','630n','605','630n');
123
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','630r','606','630r');
124
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650a','607','650a');
125
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650b',NULL,'650b');
126
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650c',NULL,'650c');
127
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650d',NULL,'650d');
128
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650v',NULL,'650v');
129
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650x',NULL,'650x');
130
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650y',NULL,'650y');
131
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650z',NULL,'650z');
132
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','651','608','651');
133
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','653a','610','653');
134
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-classification',FALSE,'','952o','995k','952o');
135
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-classification',FALSE,'',NULL,'686',NULL);
136
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'','999c','001','999c');
137
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'',NULL,'0909',NULL);
138
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itype',TRUE,'string','942c','200b','942c');
139
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itype',TRUE,'string','952y','995r','952y');
140
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('acqdate',FALSE,'date','952d','9955','952y');
141
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('place',TRUE,'string','260a','210a','260a');
142
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('publisher',TRUE,'string','260b','210c','260b');
143
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('copydate',TRUE,'date','260c',NULL,'260c'); -- No copydate for unimarc? Seems strange.
144
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('homebranch',TRUE,'string','952a','995b','952a');
145
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('holdingbranch',TRUE,'string','952b','995c','952b');
146
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('onloan',FALSE,'boolean','952q','995n','952q');
147
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itemnumber',FALSE,'number','9529','9959','9529');
148
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('issues',FALSE,'sum','952l',NULL,'952l'); -- Apparently not tracked in unimarc
149
(-)a/installer/data/mysql/kohastructure.sql (+15 lines)
Lines 931-936 CREATE TABLE `deleteditems` ( Link Here
931
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
931
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
932
932
933
--
933
--
934
-- Table structure for table `elasticsearch_mapping`
935
--
936
937
DROP TABLE IF EXISTS `elasticsearch_mapping`;
938
CREATE TABLE `elasticsearch_mapping` (
939
  `id` int(11) NOT NULL AUTO_INCREMENT,
940
  `mapping` varchar(255) DEFAULT NULL,
941
  `type` varchar(255) DEFAULT NULL,
942
  `marc21` varchar(255) DEFAULT NULL,
943
  `unimarc` varchar(255) DEFAULT NULL,
944
  `normarc` varchar(255) DEFAULT NULL,
945
  PRIMARY KEY (`id`)
946
) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8;
947
948
--
934
-- Table structure for table `ethnicity`
949
-- Table structure for table `ethnicity`
935
--
950
--
936
951
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+1 lines)
Lines 112-115 Administration: Link Here
112
              choices:
112
              choices:
113
                Solr: Solr
113
                Solr: Solr
114
                Zebra: Zebra
114
                Zebra: Zebra
115
                Elasticsearch: Elasticsearch
115
            - is the search engine used.
116
            - is the search engine used.
(-)a/koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt (-4 / +22 lines)
Lines 14-20 Link Here
14
14
15
<script type="text/javascript">
15
<script type="text/javascript">
16
  $(document).ready(function() {
16
  $(document).ready(function() {
17
    $('#bookbag_form').find("input").hide();
17
//    $('#bookbag_form').find("input").hide();
18
    $('#sort_by').change(function() {
18
    $('#sort_by').change(function() {
19
        $('#bookbag_form').submit();
19
        $('#bookbag_form').submit();
20
    });
20
    });
Lines 47-52 Link Here
47
  </div>
47
  </div>
48
  <div class="searchresults">
48
  <div class="searchresults">
49
    <form action="/cgi-bin/koha/opac-search.pl" method="get" name="bookbag_form" id="bookbag_form">
49
    <form action="/cgi-bin/koha/opac-search.pl" method="get" name="bookbag_form" id="bookbag_form">
50
        [%# IF (browse) %]
51
            <label for="browse_field">Browse: </label>
52
            <select name="browse_field" id="browse_field">
53
                <option value="title">Title</option>
54
                <option value="author">Author</option>
55
                <option value="callnumber">Call Number</option>
56
                <option value="subject">Subject</option>
57
                <option value="isbn">ISBN</option>
58
                <option value="issn">ISSN</option>
59
            </select>
60
            <input type="hidden" name="type" value="browse" />
61
            <br />
62
            <label for="search_field">Query:</label>
63
            <input type="text" name="q" style="display:initial;" />
64
            <input type="submit" value="Browse" style="display:initial;" />
65
        [%# END %]
50
      <!-- TABLE RESULTS START -->
66
      <!-- TABLE RESULTS START -->
51
      <table>
67
      <table>
52
        <thead>
68
        <thead>
Lines 76-89 Link Here
76
        </thead>
92
        </thead>
77
        <!-- Actual Search Results -->
93
        <!-- Actual Search Results -->
78
        <tbody>
94
        <tbody>
95
        [% USE Dumper %]
79
          [% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %]
96
          [% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %]
97
           [% result =SEARCH_RESULT.item('_source') %]
80
            <tr>
98
            <tr>
81
              <td>
99
              <td>
82
                <input type="checkbox" id="bib[% SEARCH_RESULT.biblionumber %]" name="biblionumber" value="[% SEARCH_RESULT.biblionumber %]" /> <label for="bib[% SEARCH_RESULT.biblionumber %]"></label>
100
                <input type="checkbox" id="bib[% result.biblionumber %]" name="biblionumber" value="[% result.biblionumber %]" /> <label for="bib[% result.biblionumber %]"></label>
83
              </td>
101
              </td>
84
              <td>
102
              <td>
85
                <a class="title" href="/cgi-bin/koha/opac-detail.pl?biblionumber=[% SEARCH_RESULT.biblionumber |url %]" title="View details for this title">[% SEARCH_RESULT.title |html %]</a>
103
                <a class="title" href="/cgi-bin/koha/opac-detail.pl?biblionumber=[% result.biblionumber |url %]" title="View details for this title">[% result.title |html %]</a>
86
                by <a href="/cgi-bin/koha/opac-search.pl?q=author:[% SEARCH_RESULT.author |url %]" title="Search for works by this author" class="author">[% SEARCH_RESULT.author %]</a>
104
                by <a href="/cgi-bin/koha/opac-search.pl?q=author:[% result.author |url %]" title="Search for works by this author" class="author">[% result.author %]</a>
87
              </td>
105
              </td>
88
            </tr>
106
            </tr>
89
          [% END %]
107
          [% END %]
(-)a/misc/search_tools/rebuild_elastic_search.pl (+148 lines)
Line 0 Link Here
1
#!/usr/bin/perl 
2
3
# This inserts records from a Koha database into elastic search
4
5
# Copyright 2014 Catalyst IT
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 3 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
=head1 NAME
23
24
rebuild_elastic_search.pl - inserts records from a Koha database into Elasticsearch
25
26
=head1 SYNOPSIS
27
28
B<rebuild_elastic_search.pl>
29
[B<-c|--commit>=C<count>]
30
[B<-v|--verbose>]
31
[B<-h|--help>]
32
[B<--man>]
33
34
=head1 DESCRIPTION
35
36
=head1 OPTIONS
37
38
=over
39
40
=item B<-c|--commit>=C<count>
41
42
Specify how many records will be batched up before they're added to Elasticsearch.
43
Higher should be faster, but will cause more RAM usage. Default is 100.
44
45
=item B<-d|--delete>
46
47
Delete the index and recreate it before indexing.
48
49
=item B<-b|--biblionumber>
50
51
Only index the supplied biblionumber, mostly for testing purposes. May be
52
repeated.
53
54
=item B<-v|--verbose>
55
56
By default, this program only emits warnings and errors. This makes it talk
57
more. Add more to make it even more wordy, in particular when debugging.
58
59
=item B<-h|--help>
60
61
Help!
62
63
=item B<--man>
64
65
Full documentation.
66
67
=cut
68
69
use autodie;
70
use Getopt::Long;
71
use Koha::Biblio;
72
use Koha::ElasticSearch::Indexer;
73
use MARC::Field;
74
use MARC::Record;
75
use Modern::Perl;
76
use Pod::Usage;
77
78
use Data::Dumper; # TODO remove
79
80
my $verbose = 0;
81
my $commit = 100;
82
my ($delete, $help, $man);
83
my (@biblionumbers);
84
85
GetOptions(
86
    'c|commit=i'       => \$commit,
87
    'd|delete'         => \$delete,
88
    'b|biblionumber=i' => \@biblionumbers,
89
    'v|verbose!'       => \$verbose,
90
    'h|help'           => \$help,
91
    'man'              => \$man,
92
);
93
94
pod2usage(1) if $help;
95
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
96
97
my $next;
98
if (@biblionumbers) {
99
    $next = sub {
100
        my $r = shift @biblionumbers;
101
        return () unless defined $r;
102
        return ($r, Koha::Biblio->get_marc_biblio($r, item_data => 1));
103
    };
104
} else {
105
    my $records = Koha::Biblio->get_all_biblios_iterator();
106
    $next = sub {
107
        $records->next();
108
    }
109
}
110
my $indexer = Koha::ElasticSearch::Indexer->new({index => 'biblios' });
111
if ($delete) {
112
    # We know it's safe to not recreate the indexer because update_index
113
    # hasn't been called yet.
114
    $indexer->delete_index();
115
}
116
117
my $count = 0;
118
my $commit_count = $commit;
119
my (@bibnums_buffer, @commit_buffer);
120
while (scalar(my ($bibnum, $rec) = $next->())) {
121
    _log(1,"$bibnum\n");
122
    $count++;
123
124
    push @bibnums_buffer, $bibnum;
125
    push @commit_buffer, $rec;
126
    if (!(--$commit_count)) {
127
        _log(2, "Committing...\n");
128
        $indexer->update_index(\@bibnums_buffer, \@commit_buffer);
129
        $commit_count = $commit;
130
        @bibnums_buffer = ();
131
        @commit_buffer = ();
132
    }
133
}
134
# There are probably uncommitted records
135
$indexer->update_index(\@bibnums_buffer, \@commit_buffer);
136
_log(1, "$count records indexed.\n");
137
138
# Output progress information.
139
#
140
#   _log($level, $msg);
141
#
142
# Will output $msg if the verbosity setting is set to $level or more. Will
143
# not include a trailing newline.
144
sub _log {
145
    my ($level, $msg) = @_;
146
147
    print $msg if ($verbose <= $level);
148
}
(-)a/myfix.txt (+3 lines)
Line 0 Link Here
1
marc_map( '245a','title' );
2
marc_map( '100a','author' );
3
marc_map( '999c','biblionumber' );
(-)a/opac/elasticsearch.pl (+102 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2013 Catalyst
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use C4::Context;
23
use CGI;
24
use C4::Auth;
25
use C4::Koha;
26
use C4::Output;
27
28
# TODO this should use the moose thing that auto-picks.
29
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
30
use Koha::ElasticSearch::Search;
31
32
my $cgi = new CGI;
33
34
my $template_name;
35
my $template_type = "basic";
36
if ( $cgi->param("idx") or $cgi->param("q") ) {
37
    $template_name = 'search/results.tt';
38
}
39
else {
40
    $template_name = 'search/advsearch.tt';
41
    $template_type = 'advsearch';
42
}
43
44
# load the template
45
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
46
    {
47
        template_name   => $template_name,
48
        query           => $cgi,
49
        type            => "opac",
50
        authnotrequired => 1,
51
    }
52
);
53
my %template_params;
54
my $format = $cgi->param("format") || 'html';
55
56
# load the Type stuff
57
my $itemtypes = GetItemTypes;
58
59
my $page = $cgi->param("page") || 1;
60
my $count =
61
     $cgi->param('count')
62
  || C4::Context->preference('OPACnumSearchResults')
63
  || 20;
64
my $q = $cgi->param("q");
65
66
my $searcher = Koha::ElasticSearch::Search->new();
67
my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
68
my $query;
69
if ($cgi->param('type') eq 'browse') {
70
    $query = $builder->build_browse_query($cgi->param('browse_field') || undef, $q );
71
    $template_params{browse} = 1;
72
} else {
73
    $query = $builder->build_query($q);
74
}
75
my $results = $searcher->search( $query, $page, $count );
76
#my $results = $searcher->search( { "match_phrase_prefix" => { "title" => "the" } } );
77
78
# This is temporary, but will do the job for now.
79
my @hits;
80
$results->each(sub {
81
        push @hits, { _source => @_[0] };
82
    });
83
# Make a list of the page numbers
84
my @pages = map { { page => $_, current => ($_ == ( $page || 1)) } } 1 .. int($results->total / $count);
85
my $max_page = int($results->total / $count);
86
# Pager template params
87
$template->param(
88
    SEARCH_RESULTS  => \@hits,
89
    PAGE_NUMBERS    => \@pages,
90
    total           => $results->total,
91
    previous_page   => ( $page > 1 ? $page - 1 : undef ),
92
    next_page       => ( $page < $max_page ? $page + 1 : undef ),
93
    follower_params => [
94
        { var => 'type',  val => $cgi->param('type') },
95
        { var => 'q',     val => $q },
96
        { var => 'count', val => $count },
97
    ],
98
    %template_params,
99
);
100
101
my $content_type = ( $format eq 'rss' or $format eq 'atom' ) ? $format : 'html';
102
output_with_http_headers $cgi, $cookie, $template->output, $content_type;
(-)a/opac/opac-search.pl (-4 / +24 lines)
Lines 28-35 use Modern::Perl; Link Here
28
# to perform, etc.
28
# to perform, etc.
29
## load Koha modules
29
## load Koha modules
30
use C4::Context;
30
use C4::Context;
31
use C4::Search;
32
33
use Data::Dumper; # TODO remove
34
35
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
36
use Koha::ElasticSearch::Search;
37
use Koha::SearchEngine::Zebra::QueryBuilder;
38
use Koha::SearchEngine::Zebra::Search;
31
39
32
my $searchengine = C4::Context->preference("SearchEngine");
40
my $searchengine = C4::Context->preference("SearchEngine");
41
my ($builder, $searcher);
42
#$searchengine = 'Zebra'; # XXX
33
for ( $searchengine ) {
43
for ( $searchengine ) {
34
    when ( /^Solr$/ ) {
44
    when ( /^Solr$/ ) {
35
        warn "We use Solr";
45
        warn "We use Solr";
Lines 37-43 for ( $searchengine ) { Link Here
37
        exit;
47
        exit;
38
    }
48
    }
39
    when ( /^Zebra$/ ) {
49
    when ( /^Zebra$/ ) {
40
50
        $builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
51
        $searcher=Koha::SearchEngine::Zebra::Search->new();
52
    }
53
    when (/^Elasticsearch$/) {
54
        # Should use the base QueryBuilder, but I don't have it wired up
55
        # for moose yet.
56
        $builder=Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
57
#        $builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
58
        $searcher=Koha::ElasticSearch::Search->new({index => 'biblios'});
41
    }
59
    }
42
}
60
}
43
61
Lines 452-458 my ($error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_ Link Here
452
my @results;
470
my @results;
453
471
454
## I. BUILD THE QUERY
472
## I. BUILD THE QUERY
455
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = buildQuery(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);
473
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = $builder->build_query_compat(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);
474
#die Dumper( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type);
456
475
457
sub _input_cgi_parse {
476
sub _input_cgi_parse {
458
    my @elements;
477
    my @elements;
Lines 522-532 if ($tag) { Link Here
522
    $pasarParams .= '&amp;simple_query=' . $simple_query;
541
    $pasarParams .= '&amp;simple_query=' . $simple_query;
523
    $pasarParams .= '&amp;query_type=' . $query_type if ($query_type);
542
    $pasarParams .= '&amp;query_type=' . $query_type if ($query_type);
524
    eval {
543
    eval {
525
        ($error, $results_hashref, $facets) = getRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);
544
        ($error, $results_hashref, $facets) = $searcher->search_compat($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);
526
    };
545
    };
527
}
546
}
547
528
# This sorts the facets into alphabetical order
548
# This sorts the facets into alphabetical order
529
if ($facets) {
549
if ($facets && @$facets) {
530
    foreach my $f (@$facets) {
550
    foreach my $f (@$facets) {
531
        $f->{facets} = [ sort { uc($a->{facet_title_value}) cmp uc($b->{facet_title_value}) } @{ $f->{facets} } ];
551
        $f->{facets} = [ sort { uc($a->{facet_title_value}) cmp uc($b->{facet_title_value}) } @{ $f->{facets} } ];
532
    }
552
    }
(-)a/t/Koha/ItemType.pm (+46 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
#
3
# Copyright 2014 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Test::More tests => 8;
23
24
BEGIN {
25
    use_ok('Koha::ItemType');
26
}
27
28
my $data = {
29
    itemtype       => 'CODE',
30
    description    => 'description',
31
    rentalcharge   => 'rentalcharge',
32
    imageurl       => 'imageurl',
33
    summary        => 'summary',
34
    checkinmsg     => 'checkinmsg',
35
    checkinmsgtype => 'checkinmsgtype',
36
};
37
38
my $type = Koha::ItemType->new($data);
39
40
is( $type->code,           'CODE',           'itemtype/code' );
41
is( $type->description,    'description',    'description' );
42
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
43
is( $type->imageurl,       'imageurl',       'imageurl' );
44
is( $type->summary,        'summary',        'summary' );
45
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
46
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );
(-)a/t/Koha_ElasticSearch.t (+25 lines)
Line 0 Link Here
1
#
2
#===============================================================================
3
#
4
#         FILE: Koha_ElasticSearch.t
5
#
6
#  DESCRIPTION: 
7
#
8
#        FILES: ---
9
#         BUGS: ---
10
#        NOTES: ---
11
#       AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
12
# ORGANIZATION: Koha Development Team
13
#      VERSION: 1.0
14
#      CREATED: 09/12/13 08:56:44
15
#     REVISION: ---
16
#===============================================================================
17
18
use strict;
19
use warnings;
20
21
use Test::More tests => 1;                      # last test to print
22
23
use_ok('Koha::ElasticSearch');
24
25
(-)a/t/Koha_ElasticSearch_Indexer.t (+51 lines)
Line 0 Link Here
1
#
2
#===============================================================================
3
#
4
#         FILE: Koha_ElasticSearch_Indexer.t
5
#
6
#  DESCRIPTION:
7
#
8
#        FILES: ---
9
#         BUGS: ---
10
#        NOTES: ---
11
#       AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
12
# ORGANIZATION: Koha Development Team
13
#      VERSION: 1.0
14
#      CREATED: 09/12/13 08:57:25
15
#     REVISION: ---
16
#===============================================================================
17
18
use strict;
19
use warnings;
20
21
use Test::More tests => 5;    # last test to print
22
use MARC::Record;
23
24
use_ok('Koha::ElasticSearch::Indexer');
25
26
my $indexer;
27
ok(
28
    my $indexer = Koha::ElasticSearch::Indexer->new(
29
        {
30
            'nodes' => ['localhost:9200'],
31
            'index' => 'mydb'
32
        }
33
    ),
34
    'Creating new indexer object'
35
);
36
37
my $marc_record = MARC::Record->new();
38
my $field = MARC::Field->new( '001', '1234567' );
39
$marc_record->append_fields($field);
40
$field = MARC::Field->new( '020', '', '', 'a' => '1234567890123' );
41
$marc_record->append_fields($field);
42
$field = MARC::Field->new( '245', '', '', 'a' => 'Title' );
43
$marc_record->append_fields($field);
44
45
my $records = [$marc_record];
46
ok( my $converted = $indexer->convert_marc_to_json($records),
47
    'Convert some records' );
48
49
is( $converted->count, 1, 'One converted record' );
50
51
ok( $indexer->update_index($records), 'Update Index' );
(-)a/t/Koha_ElasticSearch_Search.t (+39 lines)
Line 0 Link Here
1
#
2
#===============================================================================
3
#
4
#         FILE: Koha_ElasticSearch_Search.t
5
#
6
#  DESCRIPTION:
7
#
8
#        FILES: ---
9
#         BUGS: ---
10
#        NOTES: ---
11
#       AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
12
# ORGANIZATION: Koha Development Team
13
#      VERSION: 1.0
14
#      CREATED: 09/12/13 09:43:29
15
#     REVISION: ---
16
#===============================================================================
17
18
use strict;
19
use warnings;
20
21
use Test::More tests => 5;    # last test to print
22
23
use_ok('Koha::ElasticSearch::Search');
24
25
ok(
26
    my $searcher = Koha::ElasticSearch::Search->new(
27
        { 'nodes' => ['localhost:9200'], 'index' => 'mydb' }
28
    ),
29
    'Creating a Koha::ElasticSearch::Search object'
30
);
31
32
is( $searcher->index, 'mydb', 'Testing basic accessor' );
33
34
ok( $searcher->connect, 'Connect to ElasticSearch server' );
35
ok( my $results = $searcher->search( { record => 'easy' } ), 'Do a search ' );
36
37
ok( my $marcresults = $searcher->marc_search( { record => 'Fish' } ),
38
    'Do a marc search' );
39
(-)a/t/db_dependent/Koha/ItemTypes.pm (-1 / +65 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
#
3
# Copyright 2014 Catalyst IT
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
# XXX This doesn't work because I need to figure out how to do transactions
21
# in a test-case with DBIx::Class
22
23
use Modern::Perl;
24
25
use Test::More tests => 8;
26
use Data::Dumper;
27
28
BEGIN {
29
    use_ok('Koha::ItemTypes');
30
}
31
32
my $dbh = C4::Context->dbh;
33
34
# Start transaction
35
$dbh->{AutoCommit} = 0;
36
$dbh->{RaiseError} = 1;
37
38
my $prep = $dbh->prepare('INSERT INTO itemtypes (itemtype, description, rentalcharge, imageurl, summary, checkinmsg, checkinmsgtype) VALUES (?,?,?,?,?,?,?)');
39
$prep->execute('type1', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');
40
$prep->execute('type2', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');
41
42
my $itypes = Koha::ItemTypes->new();
43
44
my @types = $itypes->get_itemtype('type1', 'type2');
45
46
die Dumper(\@types);
47
my $type = $types[0];
48
ok(defined($type), 'first result');
49
is( $type->code,           'type1',           'itemtype/code' );
50
is( $type->description,    'description',    'description' );
51
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
52
is( $type->imageurl,       'imageurl',       'imageurl' );
53
is( $type->summary,        'summary',        'summary' );
54
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
55
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );
56
57
$type = $types[1];
58
ok(defined($type), 'second result');
59
is( $type->code,           'type2',           'itemtype/code' );
60
is( $type->description,    'description',    'description' );
61
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
62
is( $type->imageurl,       'imageurl',       'imageurl' );
63
is( $type->summary,        'summary',        'summary' );
64
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
65
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

Return to bug 12478