View | Details | Raw Unified | Return to bug 18948
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +1 lines)
Lines 2829-2839 sub ModZebra { Link Here
2829
            }
2829
            }
2830
        );
2830
        );
2831
        if ( $op eq 'specialUpdate' ) {
2831
        if ( $op eq 'specialUpdate' ) {
2832
            unless ($record) {
2832
            $indexer->update_index_background( [$biblionumber] );
2833
                $record = GetMarcBiblio($biblionumber, 1);
2834
            }
2835
            my $records = [$record];
2836
            $indexer->update_index_background( [$biblionumber], [$record] );
2837
        }
2833
        }
2838
        elsif ( $op eq 'recordDelete' ) {
2834
        elsif ( $op eq 'recordDelete' ) {
2839
            $indexer->delete_index_background( [$biblionumber] );
2835
            $indexer->delete_index_background( [$biblionumber] );
(-)a/Koha/SearchEngine/Elasticsearch.pm (-2 / +49 lines)
Lines 37-43 __PACKAGE__->mk_ro_accessors(qw( index )); Link Here
37
__PACKAGE__->mk_accessors(qw( sort_fields ));
37
__PACKAGE__->mk_accessors(qw( sort_fields ));
38
38
39
# Constants to refer to the standard index names
39
# Constants to refer to the standard index names
40
Readonly our $BIBLIOS_INDEX     => 'biblios';
40
Readonly our $BIBLIOS_INDEX     => 'biblios'; #These are going to be the suffixes for our aliases - most access to indexes will be via alias
41
Readonly our $AUTHORITIES_INDEX => 'authorities';
41
Readonly our $AUTHORITIES_INDEX => 'authorities';
42
42
43
=head1 NAME
43
=head1 NAME
Lines 236-241 sub get_elasticsearch_mappings { Link Here
236
    return $mappings;
236
    return $mappings;
237
}
237
}
238
238
239
=head2 get_alias_index
240
241
Get the ES specific index names that an alias points to
242
243
Returns an index name or undef
244
245
=cut
246
247
sub get_alias_index {
248
    my ($self) = @_;
249
    my $alias = $self->store->{index_name};
250
    if ( $self->store->es->indices->exists_alias( name => $alias ) ) { #use the embedded es object to access ES methods directly and check if alias points to anything
251
        my $current_index_info = $self->store->es->indices->get_alias( name => $alias ); #we get a hashref of info
252
        my @current_indices = keys %$current_index_info; #the keys contain the names of indexes pointed to be the alias
253
        return $current_indices[0]; #For now we are maintaining a 1:1 ratio, pointing to several is possible
254
    } else {
255
        return undef;
256
    }
257
}
258
259
=head2 update_alias_indices
260
261
Takes our ES alias name and two index names and swaps the old for the new
262
return
263
264
=cut
265
266
sub update_alias {
267
    my $self = shift;
268
    my ( $params ) = @_;
269
    my $alias = $self->get_elasticsearch_params()->{index_name}; #FIXME this should probably be an object property/method
270
    die "Must supply old_index and new_index parameters" unless ( $params->{old_index} && $params->{new_index} );
271
    return $self->store->es->indices->update_aliases( #this is atomic so we won't delete unless this works
272
        body => {
273
            actions => [
274
                { add    => { alias => $alias, index => $params->{new_index} } },
275
                { remove => { alias => $alias, index => $params->{old_index} } }
276
            ]
277
        }
278
    );
279
}
280
239
=head2 _elasticsearch_mapping_for_*
281
=head2 _elasticsearch_mapping_for_*
240
282
241
Get the ES mappings for the given data type or a special mapping case
283
Get the ES mappings for the given data type or a special mapping case
Lines 418-427 These are of a form suited to Catmandu's MARC fixers. Link Here
418
sub _foreach_mapping {
460
sub _foreach_mapping {
419
    my ( $self, $sub ) = @_;
461
    my ( $self, $sub ) = @_;
420
462
463
    my $index_type;
464
465
    $self->index =~ /biblio/ ? $index_type = 'biblios' :
466
        $self->index =~ /authorities/ ? $index_type = 'authorities' : return;
467
421
    # TODO use a caching framework here
468
    # TODO use a caching framework here
422
    my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
469
    my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
423
        {
470
        {
424
            'search_marc_map.index_name' => $self->index,
471
            'search_marc_map.index_name' => $index_type,
425
        },
472
        },
426
        {   join => { search_marc_to_fields => 'search_marc_map' },
473
        {   join => { search_marc_to_fields => 'search_marc_map' },
427
            '+select' => [
474
            '+select' => [
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-19 / +93 lines)
Lines 20-26 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here
20
use Carp;
20
use Carp;
21
use Modern::Perl;
21
use Modern::Perl;
22
use base qw(Koha::SearchEngine::Elasticsearch);
22
use base qw(Koha::SearchEngine::Elasticsearch);
23
use Data::Dumper;
23
24
use Koha::BiblioUtils;
25
use Koha::MetadataRecord::Authority;
24
26
25
# For now just marc, but we can do anything here really
27
# For now just marc, but we can do anything here really
26
use Catmandu::Importer::MARC;
28
use Catmandu::Importer::MARC;
Lines 89-95 sub update_index { Link Here
89
91
90
    my $from    = $self->_convert_marc_to_json($records);
92
    my $from    = $self->_convert_marc_to_json($records);
91
93
92
    print Data::Dumper::Dumper( $from->to_array );
94
    #print Data::Dumper::Dumper( $from->to_array );
93
    $self->store->bag->add_many($from);
95
    $self->store->bag->add_many($from);
94
    $self->store->bag->commit;
96
    $self->store->bag->commit;
95
    return 1;
97
    return 1;
Lines 110-116 it to be updated by a regular index cron job in the future. Link Here
110
112
111
sub update_index_background {
113
sub update_index_background {
112
    my $self = shift;
114
    my $self = shift;
113
    $self->update_index(@_);
115
    warn Data::Dumper::Dumper( @_ );
116
    $self->do_reindex(undef, undef, 0, @_);
114
}
117
}
115
118
116
=head2 $indexer->delete_index($biblionums)
119
=head2 $indexer->delete_index($biblionums)
Lines 142-149 sub delete_index_background { Link Here
142
145
143
=head2 $indexer->drop_index();
146
=head2 $indexer->drop_index();
144
147
145
Drops the index from the elasticsearch server. Calling C<update_index>
148
Drops the index from the elasticsearch server.
146
after this will recreate it again.
149
We need to create a new object once this has been done.
147
150
148
=cut
151
=cut
149
152
Lines 151-177 sub drop_index { Link Here
151
    my ($self) = @_;
154
    my ($self) = @_;
152
155
153
    $self->store->drop();
156
    $self->store->drop();
154
    my $params  = $self->get_elasticsearch_params();
155
    $self->store(
156
        Catmandu::Store::ElasticSearch->new(
157
            %$params,
158
            index_settings => $self->get_elasticsearch_settings(),
159
            index_mappings => $self->get_elasticsearch_mappings(),
160
        )
161
    );
162
}
157
}
163
158
164
159
165
=head2 $indexer->do_reindex();
160
=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers);
161
162
Performs a reindex, full or patial is based on whether biblionumbers are passed in
163
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
164
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
166
165
167
Performs a reindex based on records passed in
166
C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case.
167
168
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
169
170
C<$verbose> specifies warning level - only for command line use currently.
171
172
C<$biblionums> is an arrayref containing the biblionumbers for the records.
168
173
169
=cut
174
=cut
170
175
171
sub do_reindex {
176
sub do_reindex {
172
    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
177
    my ($self, $delete, $commit, $verbose, $biblionumbers) = @_;
173
178
174
    _log(1, $verbose, "Indexing ".$self->index."\n");
179
    _log(1, $verbose, "Indexing ".$self->index."\n");
180
    my $next;
175
181
176
    if ( scalar @$biblionumbers ) {
182
    if ( scalar @$biblionumbers ) {
177
        if ( $self->index eq 'biblios' ) {
183
        if ( $self->index eq 'biblios' ) {
Lines 197-204 sub do_reindex { Link Here
197
        $next = sub { $records->next(); };
203
        $next = sub { $records->next(); };
198
    }
204
    }
199
205
200
warn 
206
    my $current_index = $self->get_alias_index();
201
    $self->drop_index() if ( $delete );
207
208
    if ( scalar @$biblionumbers && !$delete && $current_index) {
209
210
        #In this one case we index onto existing alias
211
        $self->_index_records( $next, $commit, $verbose );
212
213
    } else {
214
215
        #In all other cases we are going to use a new index then switch our alias to this index
216
        #'time' is used for providing a unique name
217
        my $alias_name = $self->store->{index_name};
218
219
        my $new_indexer =
220
            Koha::SearchEngine::Elasticsearch::Indexer->new( {
221
                index => $self->index.time 
222
            });
223
224
        $new_indexer->_index_records( $next, $commit, $verbose );
225
226
        if ( $current_index ){                   #If the alias already exists
227
            $self->update_alias({                #let's point the alias to the new index
228
                old_index => $current_index,
229
                new_index => $new_indexer->store->{index_name}
230
            });
231
            $current_index =~ s/koha_kohadev_//; #Then we need short index name to build object
232
233
            my $old_index =
234
                Koha::SearchEngine::Elasticsearch::Indexer->new({
235
                    index => $current_index
236
                });
237
238
            $old_index->drop_index();            #And we drop the specific index that the alias pointed to originally
239
        } else {                                 #The alias was not created
240
            $self->drop_index();                 #Drop the index we created initially
241
                                                 #It will have the same name as our alias so must be dropped first
242
                                                 #It should be empty (we indexed into a new one)
243
            $new_indexer->store->es->indices->put_alias(
244
                name => $alias_name,
245
                index => $new_indexer->store->{index_name}
246
            );
247
        }
248
249
    }
250
}
251
252
=head2 $indexer->index_records( $next, $commit, $verbose );
253
254
Performs a reindex, full or patial is based on whether biblionumbers are passed in
255
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
256
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
257
258
C<$next> is a function reference, used for iterating over records.
259
260
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
261
262
C<$verbose> specifies warning level - only for command line use currently.
263
264
=cut
265
266
sub _index_records {
267
    my ( $self, $next, $commit, $verbose ) = @_;
268
    $commit ||= 5000;
202
269
203
    my $count        = 0;
270
    my $count        = 0;
204
    my $commit_count = $commit;
271
    my $commit_count = $commit;
Lines 219-229 warn Link Here
219
            @commit_buffer = ();
286
            @commit_buffer = ();
220
        }
287
        }
221
    }
288
    }
222
    # There are probably uncommitted records (beacuse we didn't hit exact commit count)
289
    # There are probably uncommitted records (because we didn't hit exact commit count)
223
    $self->update_index( \@id_buffer, \@commit_buffer );
290
    $self->update_index( \@id_buffer, \@commit_buffer );
224
    _log( 1, $verbose, "$count records indexed.\n" );
291
    _log( 1, $verbose, "$count records indexed.\n" );
225
}
292
}
226
293
294
=head2 $indexer->_log( $level, $verbose, $msg )
295
296
Internal function, print msg to screen if verbosity > level
297
298
=cut
227
299
228
sub _log {
300
sub _log {
229
    my ($level, $verbose, $msg) = @_;
301
    my ($level, $verbose, $msg) = @_;
Lines 273-276 __END__ Link Here
273
345
274
=item Robin Sheat C<< <robin@catalyst.net.nz> >>
346
=item Robin Sheat C<< <robin@catalyst.net.nz> >>
275
347
348
=item Nick Clemens C<< <nick@bywatersolutions.com> >>
349
276
=back
350
=back
(-)a/misc/search_tools/rebuild_elastic_search.pl (-10 / +5 lines)
Lines 92-99 use MARC::Record; Link Here
92
use Modern::Perl;
92
use Modern::Perl;
93
use Pod::Usage;
93
use Pod::Usage;
94
94
95
use Data::Dumper; # TODO remove
96
97
my $verbose = 0;
95
my $verbose = 0;
98
my $commit = 5000;
96
my $commit = 5000;
99
my ($delete, $help, $man);
97
my ($delete, $help, $man);
Lines 103-111 my (@biblionumbers); Link Here
103
GetOptions(
101
GetOptions(
104
    'c|commit=i'       => \$commit,
102
    'c|commit=i'       => \$commit,
105
    'd|delete'         => \$delete,
103
    'd|delete'         => \$delete,
106
    'a|authorities' => \$index_authorities,
104
    'a|authorities'    => \$index_authorities,
107
    'b|biblios' => \$index_biblios,
105
    'b|biblios'        => \$index_biblios,
108
    'bn|bnumber=i' => \@biblionumbers,
106
    'bn|bnumber=i'     => \@biblionumbers,
109
    'v|verbose+'       => \$verbose,
107
    'v|verbose+'       => \$verbose,
110
    'h|help'           => \$help,
108
    'h|help'           => \$help,
111
    'man'              => \$man,
109
    'man'              => \$man,
Lines 121-137 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here
121
119
122
sanity_check();
120
sanity_check();
123
121
124
my $next;
125
print "$verbose verbosity";
126
if ($index_biblios) {
122
if ($index_biblios) {
127
    my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
123
    my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
128
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
124
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
129
    $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers);
125
    $indexer->do_reindex($delete,$commit,$verbose,\@biblionumbers);
130
}
126
}
131
if ($index_authorities) {
127
if ($index_authorities) {
132
    my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
128
    my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
133
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
129
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
134
    $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers);
130
    $indexer->do_reindex($delete,$commit,$verbose, \@biblionumbers);
135
}
131
}
136
132
137
# Checks some basic stuff to ensure that it's sane before we start.
133
# Checks some basic stuff to ensure that it's sane before we start.
138
- 

Return to bug 18948