View | Details | Raw Unified | Return to bug 35086
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-6 / +19 lines)
Lines 19-27 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here
19
19
20
use Carp qw( carp croak );
20
use Carp qw( carp croak );
21
use Modern::Perl;
21
use Modern::Perl;
22
use Try::Tiny qw( catch try );
22
use Try::Tiny       qw( catch try );
23
use List::Util qw( any );
23
use List::Util      qw( any );
24
use base qw(Koha::SearchEngine::Elasticsearch);
24
use List::MoreUtils qw( natatime );
25
use base            qw(Koha::SearchEngine::Elasticsearch);
25
26
26
use Koha::Exceptions;
27
use Koha::Exceptions;
27
use Koha::Exceptions::Elasticsearch;
28
use Koha::Exceptions::Elasticsearch;
Lines 309-314 at the moment. Link Here
309
The other variables are used for parity with Zebra indexing calls. Currently the calls are passed through
310
The other variables are used for parity with Zebra indexing calls. Currently the calls are passed through
310
to Zebra as well.
311
to Zebra as well.
311
312
313
Will obey the chunk_size defined in koha-conf for amount of records to send during a single reindex, or default
314
to 5000.
315
312
=cut
316
=cut
313
317
314
sub index_records {
318
sub index_records {
Lines 316-325 sub index_records { Link Here
316
    $record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;
320
    $record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;
317
    $records = [$records] if ref $records ne 'ARRAY' && defined $records;
321
    $records = [$records] if ref $records ne 'ARRAY' && defined $records;
318
    if ( $op eq 'specialUpdate' ) {
322
    if ( $op eq 'specialUpdate' ) {
319
        if ($records){
323
        my $config    = $self->get_elasticsearch_params;
320
            $self->update_index( $record_numbers, $records );
324
        my $at_a_time = $config->{chunk_size} // 5000;
325
        my ( $record_chunks, $record_id_chunks );
326
        $record_chunks    = natatime $at_a_time, @$records        if ($records);
327
        $record_id_chunks = natatime $at_a_time, @$record_numbers if ($record_numbers);
328
        if ($records) {
329
            while ( (my @records = $record_chunks->()) && (my @record_ids = $record_id_chunks->()) ) {
330
                $self->update_index( \@record_ids, \@records );
331
            }
321
        } else {
332
        } else {
322
            $self->update_index_background( $record_numbers, $server );
333
            while ( my @record_ids = $record_id_chunks->() ) {
334
                $self->update_index_background( \@record_ids, $server );
335
            }
323
        }
336
        }
324
    }
337
    }
325
    elsif ( $op eq 'recordDelete' ) {
338
    elsif ( $op eq 'recordDelete' ) {
(-)a/etc/koha-conf.xml (+2 lines)
Lines 171-176 Link Here
171
     <cxn_pool>Static</cxn_pool>
171
     <cxn_pool>Static</cxn_pool>
172
     <!-- See https://metacpan.org/pod/Search::Elasticsearch#trace_to -->
172
     <!-- See https://metacpan.org/pod/Search::Elasticsearch#trace_to -->
173
     <!-- <trace_to>Stderr</trace_to> -->
173
     <!-- <trace_to>Stderr</trace_to> -->
174
     <!-- You can specify the maximum chunk size for records when batch processing in Koha -->
175
     <!-- <chunk_size>500</chunk_size> -->
174
 </elasticsearch>
176
 </elasticsearch>
175
 <!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->
177
 <!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->
176
 <!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->
178
 <!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Indexer.t (-2 / +19 lines)
Lines 88-94 subtest 'create_index() tests' => sub { Link Here
88
};
88
};
89
89
90
subtest 'index_records() tests' => sub {
90
subtest 'index_records() tests' => sub {
91
    plan tests => 2;
91
    plan tests => 4;
92
    my $mock_index = Test::MockModule->new("Koha::SearchEngine::Elasticsearch::Indexer");
92
    my $mock_index = Test::MockModule->new("Koha::SearchEngine::Elasticsearch::Indexer");
93
    $mock_index->mock( update_index => sub {
93
    $mock_index->mock( update_index => sub {
94
        my ($self, $record_ids, $records) = @_;
94
        my ($self, $record_ids, $records) = @_;
Lines 122-127 subtest 'index_records() tests' => sub { Link Here
122
    "Update background " . $biblio->biblionumber,
122
    "Update background " . $biblio->biblionumber,
123
    "When passing id only to index_records the marc record is fetched and passed through to update_index";
123
    "When passing id only to index_records the marc record is fetched and passed through to update_index";
124
124
125
    my $chunks = 0;
126
    $mock_index->mock(
127
        update_index => sub {
128
            my ( $self, $record_ids, $records ) = @_;
129
            $chunks++;
130
        }
131
    );
132
133
    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo' } );
134
    my @big_array = 1 .. 10000;
135
    $indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
136
    is( $chunks, 2, "We split 10000 records into two chunks when chunk size not set" );
137
138
    $chunks = 0;
139
    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo', chunk_size => 10 } );
140
    $indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
141
    is( $chunks, 1000, "We split 10000 records into 1000 chunks when chunk size is 10" );
142
125
};
143
};
126
144
127
subtest 'update_index' => sub {
145
subtest 'update_index' => sub {
128
- 

Return to bug 35086