Attachment 82582 Details for Bug 21872 – Bug 21872: Elasticsearch indexing faster by making it multi-threaded

[patch] Bug 21872: Elasticsearch indexing faster by making it multi-threaded

Bug-21872-Elasticsearch-indexing-faster-by-making-.patch (text/plain), 5.48 KB, created by David Gustafsson on 2018-11-22 16:58:17 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Gustafsson

Created: 2018-11-22 16:58:17 UTC

Size: 5.48 KB

patch

obsolete

>From 83ea1a9bbe4a234ee7d02e0d0eb65b3eedbea649 Mon Sep 17 00:00:00 2001
>From: David Gustafsson <david.gustafsson@ub.gu.se>
>Date: Thu, 22 Nov 2018 17:55:49 +0100
>Subject: [PATCH] Bug 21872: Elasticsearch indexing faster by making it
> multi-threaded
>
>Add record_batches script for generating biblionumber bathes and
>add --start-bnumber and --end-bnumber to rebuild_elastic_search.pl
>script.
>---
> Koha/BiblioUtils.pm                         | 20 +++++++--
> misc/record_batches.pl                      | 68 +++++++++++++++++++++++++++++
> misc/search_tools/rebuild_elastic_search.pl | 29 ++++++++++--
> 3 files changed, 110 insertions(+), 7 deletions(-)
> create mode 100644 misc/record_batches.pl
>
>diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm
>index 73fc12aa4b..2676428939 100644
>--- a/Koha/BiblioUtils.pm
>+++ b/Koha/BiblioUtils.pm
>@@ -111,11 +111,25 @@ The iterator is a Koha::MetadataIterator object.
> =cut
> 
> sub get_all_biblios_iterator {
>+    my ($self, $params) = @_;
>     my $database = Koha::Database->new();
>     my $schema   = $database->schema();
>-    my $rs =
>-      $schema->resultset('Biblio')->search( {},
>-        { columns => [qw/ biblionumber /] } );
>+    $params //= {};
>+
>+    my $conditions = {};
>+
>+    if ($params->{start_biblionumber}) {
>+        $conditions->{biblionumber}->{'>='} = $params->{start_biblionumber};
>+    }
>+
>+    if ($params->{end_biblionumber}) {
>+        $conditions->{biblionumber}->{'<='} = $params->{end_biblionumber};
>+    }
>+
>+    my $rs = $schema->resultset('Biblio')->search(
>+        $conditions,
>+        { columns => [qw/ biblionumber /] }
>+    );
>     my $next_func = sub {
>         # Warn and skip bad records, otherwise we break the loop
>         while (1) {
>diff --git a/misc/record_batches.pl b/misc/record_batches.pl
>new file mode 100644
>index 0000000000..e003c8ef5c
>--- /dev/null
>+++ b/misc/record_batches.pl
>@@ -0,0 +1,68 @@
>+#!/usr/bin/perl
>+
>+use Modern::Perl;
>+use C4::Context;
>+use Getopt::Long;
>+use Pod::Usage;
>+
>+my $gt_biblionumber;
>+my $batch_size;
>+my $help;
>+GetOptions(
>+    'gt_biblionumber=i' => \$gt_biblionumber,
>+    'batch_size=i' => \$batch_size,
>+    'h|help|?' => \$help
>+);
>+if ($help) {
>+    pod2usage(1);
>+}
>+
>+my $dbh = C4::Context->dbh;
>+my $step = $batch_size || 25000;
>+my $offset = 0;
>+my $prev_biblionumber;
>+my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };
>+my @ranges;
>+my $biblionumber;
>+
>+while (1) {
>+    my $query = $gt_biblionumber ?
>+        qq{SELECT `biblionumber` FROM `biblio` WHERE `biblionumber` >= $gt_biblionumber ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset} :
>+        qq{SELECT `biblionumber` FROM `biblio` ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset};
>+    my ($biblionumber) = @{ $dbh->selectcol_arrayref($query) };
>+    if (!$biblionumber) {
>+        push @ranges, [$prev_biblionumber, $max_biblionumber];
>+        last;
>+    }
>+    elsif ($prev_biblionumber) {
>+        push @ranges, [$prev_biblionumber, $biblionumber - 1];
>+    }
>+    $prev_biblionumber = $biblionumber;
>+    $offset += $step;
>+}
>+
>+foreach my $range (@ranges) {
>+    print join(' ', @{$range}) . "\n"; # @TODO: Option for argument and range separtor?
>+}
>+
>+=head1 NAME
>+
>+record batches - This script outputs ranges of biblionumbers
>+
>+=head1 SYNOPSIS
>+
>+record_batches.pl [-h|--help] [--gt_biblionumber=BIBLIONUMBER]
>+
>+=head1 OPTIONS
>+
>+=over
>+
>+=item B<-h|--help>
>+
>+Print a brief help message.
>+
>+=item B<--gt_biblionumber>
>+
>+ --gt_biblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER
>+
>+=item B<--batch_size>
>diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl
>index 63065cad79..1b75749954 100755
>--- a/misc/search_tools/rebuild_elastic_search.pl
>+++ b/misc/search_tools/rebuild_elastic_search.pl
>@@ -64,6 +64,15 @@ Only index the supplied biblionumber, mostly for testing purposes. May be
> repeated. This also applies to authorities via authid, so if you're using it,
> you probably only want to do one or the other at a time.
> 
>+=item B<-sbn|--start-bnumber>
>+
>+Only index biblios with biblionumber greater or equal than supplied
>+biblionumber.
>+
>+=item B<-ebn|--end-bnumber>
>+
>+Only index biblios with biblionumber less or equal to supplied biblionumber.
>+
> =item B<-v|--verbose>
> 
> By default, this program only emits warnings and errors. This makes it talk
>@@ -94,9 +103,16 @@ use Pod::Usage;
> 
> my $verbose = 0;
> my $commit = 5000;
>-my ($delete, $help, $man);
>-my ($index_biblios, $index_authorities);
>-my (@biblionumbers);
>+my (
>+    $delete,
>+    $help,
>+    $man,
>+    $index_biblios,
>+    $index_authorities,
>+    @biblionumbers,
>+    $start_biblionumber,
>+    $end_biblionumber
>+);
> 
> $|=1; # flushes output
> 
>@@ -106,6 +122,8 @@ GetOptions(
>     'a|authorities' => \$index_authorities,
>     'b|biblios' => \$index_biblios,
>     'bn|bnumber=i' => \@biblionumbers,
>+    'sbn|start-bnumber=i' => \$start_biblionumber,
>+    'ebn|end-bnumber=i' => \$end_biblionumber,
>     'v|verbose+'       => \$verbose,
>     'h|help'           => \$help,
>     'man'              => \$man,
>@@ -131,7 +149,10 @@ if ($index_biblios) {
>             return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
>         };
>     } else {
>-        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
>+        my $records = Koha::BiblioUtils->get_all_biblios_iterator({
>+            'start_biblionumber' => $start_biblionumber,
>+            'end_biblionumber' => $end_biblionumber,
>+        });
>         $next = sub {
>             $records->next();
>         }
>-- 
>2.11.0

Actions: View | Diff | Splinter Review

Attachments on bug 21872: 82582 | 82583 | 82584 | 82585 | 82586 | 82598 | 82599 | 82780 | 85119 | 85121 | 85975 | 85976 | 85977 | 85978 | 85979 | 86053 | 86054 | 86055 | 86056 | 86057 | 89109 | 89110 | 89111 | 89112 | 89113