From a2bf434d0c40f7c7bedd367a339d762dcdccc6db Mon Sep 17 00:00:00 2001 From: Tomas Cohen Arazi Date: Mon, 21 Apr 2025 15:16:32 -0300 Subject: [PATCH] Bug 39694: Make `es_indexer_daemon.pl` use batch_size to limit DB poll size This patch makes the indexer poll the DB in configurable batches. Pretty much like Rabbit does. The only change is that it will now pass a LIMIT to the query. To test: 0. Have KTD started with ES (skip if you have an instance running): $ ktd --es7 up -d 1. Stop the indexer and rabbit: $ ktd --shell k$ sudo koha-es-indexer --stop kohadev k$ sudo service rabbitmq-server stop 2. On a separate terminal, watch the queue: k$ watch -n 0.5 'echo "SELECT COUNT(*) FROM background_jobs WHERE status=\"new\"" | koha-mysql kohadev' => SUCCESS: It should be empty 3. Add a bunch of things to index (this works with default KTD data): k$ for i in $(seq 1 438; seq 1 438) ; do echo "INSERT INTO background_jobs (status,queue,data) VALUES ('new','elastic_index','{\"record_server\":\"biblioserver\",\"record_ids\":[$i]}')" | koha-mysql kohadev ; done 4. Start the ES indexer: k$ sudo koha-es-indexer --start kohadev 5. Look at the queue size => FAIL: It shrinks too fast [1] 6. Apply this patch 7. Restart the indexer k$ perl misc/workers/es_indexer_daemon.pl & 8. Repeat 3 9. Look at the queue size: => SUCCESS: It shrinks 'slowly' 10. Sign off :-D [1] This is a problem in certain scenarios, not in all of them. This is the best I come up with in order to test the behavior change. Anyone can set a new size of their taste. Signed-off-by: Tomas Cohen Arazi --- misc/workers/es_indexer_daemon.pl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misc/workers/es_indexer_daemon.pl b/misc/workers/es_indexer_daemon.pl index aa2f2a9db26..24a54d52f68 100755 --- a/misc/workers/es_indexer_daemon.pl +++ b/misc/workers/es_indexer_daemon.pl @@ -64,7 +64,8 @@ use Koha::BackgroundJobs; use Koha::SearchEngine; use Koha::SearchEngine::Indexer; -my ( $help, $batch_size ); +my $help; +my $batch_size = 10; my $not_found_retries = {}; my $max_retries = $ENV{MAX_RETRIES} || 10; @@ -76,8 +77,6 @@ GetOptions( pod2usage(0) if $help; -$batch_size //= 10; - warn "Not using Elasticsearch" unless C4::Context->preference('SearchEngine') eq 'Elasticsearch'; my $logger = Koha::Logger->get( { interface => 'worker' } ); @@ -172,7 +171,10 @@ while (1) { } } else { - @jobs = Koha::BackgroundJobs->search( { status => 'new', queue => 'elastic_index' } )->as_list; + @jobs = Koha::BackgroundJobs->search( + { status => 'new', queue => 'elastic_index' }, + { rows => $batch_size } + )->as_list; commit(@jobs); @jobs = (); sleep 10; -- 2.49.0