Attachment #160853 for bug #35086

View | Details | Raw Unified | Return to bug 35086
Collapse All | Expand All

Lines 19-27 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-6 / +19 lines)
19		19
20	use Carp qw( carp croak );	20	use Carp qw( carp croak );
21	use Modern::Perl;	21	use Modern::Perl;
22	use Try::Tiny qw( catch try );	22	use Try::Tiny qw( catch try );
23	use List::Util qw( any );	23	use List::Util qw( any );
24	use base qw(Koha::SearchEngine::Elasticsearch);	24	use List::MoreUtils qw( natatime );
		25	use base qw(Koha::SearchEngine::Elasticsearch);
25		26
26	use Koha::Exceptions;	27	use Koha::Exceptions;
27	use Koha::Exceptions::Elasticsearch;	28	use Koha::Exceptions::Elasticsearch;
Lines 309-314 at the moment. Link Here
309	The other variables are used for parity with Zebra indexing calls. Currently the calls are passed through	310	The other variables are used for parity with Zebra indexing calls. Currently the calls are passed through
310	to Zebra as well.	311	to Zebra as well.
311		312
		313	Will obey the chunk_size defined in koha-conf for amount of records to send during a single reindex, or default
		314	to 5000.
		315
312	=cut	316	=cut
313		317
314	sub index_records {	318	sub index_records {
Lines 316-325 sub index_records { Link Here
316	$record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;	320	$record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;
317	$records = [$records] if ref $records ne 'ARRAY' && defined $records;	321	$records = [$records] if ref $records ne 'ARRAY' && defined $records;
318	if ( $op eq 'specialUpdate' ) {	322	if ( $op eq 'specialUpdate' ) {
319	if ($records){	323	my $config = $self->get_elasticsearch_params;
320	$self->update_index( $record_numbers, $records );	324	my $at_a_time = $config->{chunk_size} // 5000;
		325	my ( $record_chunks, $record_id_chunks );
		326	$record_chunks = natatime $at_a_time, @$records if ($records);
		327	$record_id_chunks = natatime $at_a_time, @$record_numbers if ($record_numbers);
		328	if ($records) {
		329	while ( (my @records = $record_chunks->()) && (my @record_ids = $record_id_chunks->()) ) {
		330	$self->update_index( \@record_ids, \@records );
		331	}
321	} else {	332	} else {
322	$self->update_index_background( $record_numbers, $server );	333	while ( my @record_ids = $record_id_chunks->() ) {
		334	$self->update_index_background( \@record_ids, $server );
		335	}
323	}	336	}
324	}	337	}
325	elsif ( $op eq 'recordDelete' ) {	338	elsif ( $op eq 'recordDelete' ) {

Lines 171-176 Link Here

(-)a/etc/koha-conf.xml (+2 lines)
171	<cxn_pool>Static</cxn_pool>	171	<cxn_pool>Static</cxn_pool>
172	<!-- See https://metacpan.org/pod/Search::Elasticsearch#trace_to -->	172	<!-- See https://metacpan.org/pod/Search::Elasticsearch#trace_to -->
173	<!-- <trace_to>Stderr</trace_to> -->	173	<!-- <trace_to>Stderr</trace_to> -->
		174	<!-- You can specify the maximum chunk size for records when batch processing in Koha -->
		175	<!-- <chunk_size>500</chunk_size> -->
174	</elasticsearch>	176	</elasticsearch>
175	<!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->	177	<!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->
176	<!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->	178	<!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->




};

subtest 'index_records() tests' => sub {
    plan tests => 4;
    my $mock_index = Test::MockModule->new("Koha::SearchEngine::Elasticsearch::Indexer");
    $mock_index->mock( update_index => sub {
        my ($self, $record_ids, $records) = @_;

    "Update background " . $biblio->biblionumber,
    "When passing id only to index_records the marc record is fetched and passed through to update_index";

    my $chunks = 0;
    $mock_index->mock(
        update_index => sub {
            my ( $self, $record_ids, $records ) = @_;
            $chunks++;
        }
    );

    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo' } );
    my @big_array = 1 .. 10000;
    $indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
    is( $chunks, 2, "We split 10000 records into two chunks when chunk size not set" );

    $chunks = 0;
    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo', chunk_size => 10 } );
    $indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
    is( $chunks, 1000, "We split 10000 records into 1000 chunks when chunk size is 10" );

};

subtest 'update_index' => sub {
- 

Return to bug 35086

Lines 88-94 subtest 'create_index() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Indexer.t (-2 / +19 lines)
88	};	88	};
89		89
90	subtest 'index_records() tests' => sub {	90	subtest 'index_records() tests' => sub {
91	plan tests => 2;	91	plan tests => 4;
92	my $mock_index = Test::MockModule->new("Koha::SearchEngine::Elasticsearch::Indexer");	92	my $mock_index = Test::MockModule->new("Koha::SearchEngine::Elasticsearch::Indexer");
93	$mock_index->mock( update_index => sub {	93	$mock_index->mock( update_index => sub {
94	my ($self, $record_ids, $records) = @_;	94	my ($self, $record_ids, $records) = @_;
Lines 122-127 subtest 'index_records() tests' => sub { Link Here
122	"Update background " . $biblio->biblionumber,	122	"Update background " . $biblio->biblionumber,
123	"When passing id only to index_records the marc record is fetched and passed through to update_index";	123	"When passing id only to index_records the marc record is fetched and passed through to update_index";
124		124
		125	my $chunks = 0;
		126	$mock_index->mock(
		127	update_index => sub {
		128	my ( $self, $record_ids, $records ) = @_;
		129	$chunks++;
		130	}
		131	);
		132
		133	t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo' } );
		134	my @big_array = 1 .. 10000;
		135	$indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
		136	is( $chunks, 2, "We split 10000 records into two chunks when chunk size not set" );
		137
		138	$chunks = 0;
		139	t::lib::Mocks::mock_config( 'elasticsearch', { server => 'false', index_name => 'pseudo', chunk_size => 10 } );
		140	$indexer->index_records( \@big_array, 'specialUpdate', 'biblioserver', \@big_array );
		141	is( $chunks, 1000, "We split 10000 records into 1000 chunks when chunk size is 10" );
		142
125	};	143	};
126		144
127	subtest 'update_index' => sub {	145	subtest 'update_index' => sub {
128	-