Attachment 82598 Details for Bug 21872 – Bug 21872: Add slice parameter to rebuild_elastic_search.pl

[patch] Bug 21872: Add slice parameter to rebuild_elastic_search.pl

Bug-21872-Add-slice-parameter-to-rebuildelasticsea.patch (text/plain), 10.09 KB, created by Ere Maijala on 2018-11-23 08:44:06 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2018-11-23 08:44:06 UTC

Size: 10.09 KB

patch

obsolete

>From 7b9413f8e1f601961dca9d8d87248292e6d25e12 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Fri, 23 Nov 2018 10:43:34 +0200
>Subject: [PATCH] Bug 21872: Add slice parameter to rebuild_elastic_search.pl
>
>The slice parameter allows one to define a slice of the records to index for parallel processing.
>---
> Koha/BiblioUtils.pm                         | 29 ++++++++-
> Koha/MetadataRecord/Authority.pm            | 91 +++++++++++++++++++----------
> misc/search_tools/rebuild_elastic_search.pl | 42 ++++++++-----
> 3 files changed, 116 insertions(+), 46 deletions(-)
>
>diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm
>index 73fc12a..82f7858 100644
>--- a/Koha/BiblioUtils.pm
>+++ b/Koha/BiblioUtils.pm
>@@ -101,29 +101,54 @@ sub get_from_biblionumber {
> 
> =head2 get_all_biblios_iterator
> 
>-    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
>+    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
> 
> This will provide an iterator object that will, one by one, provide the
> Koha::BiblioUtils of each biblio. This will include the item data.
> 
> The iterator is a Koha::MetadataIterator object.
> 
>+Possible options are:
>+
>+=over 4
>+
>+=item C<slice>
>+
>+slice may be defined as a hash of two values: index and count. index
>+is the slice number to process and count is total number of slices. 
>+With this information the iterator returns just the given slice of
>+records instead of all.
>+
> =cut
> 
> sub get_all_biblios_iterator {
>+    my ($self, %options) = @_;
>+
>+    my ($slice_modulo, $slice_count);
>+    if ($options{slice}) {
>+        $slice_count = $options{slice}->{count};
>+        $slice_modulo = $options{slice}->{index};
>+        $slice_modulo = 0 if ($slice_modulo == $slice_count);
>+    }
>+
>     my $database = Koha::Database->new();
>     my $schema   = $database->schema();
>     my $rs =
>       $schema->resultset('Biblio')->search( {},
>         { columns => [qw/ biblionumber /] } );
>+
>     my $next_func = sub {
>-        # Warn and skip bad records, otherwise we break the loop
>         while (1) {
>             my $row = $rs->next();
>             return if !$row;
>+            # Check if biblionumber matches our slice definition
>+            if (defined $slice_count) {
>+                next unless $row->biblionumber % $slice_count == $slice_modulo;
>+            }
>             my $marc = C4::Biblio::GetMarcBiblio({
>                 biblionumber => $row->biblionumber,
>                 embed_items  => 1 });
>+            # Warn and skip bad records, otherwise we break the loop
>             my $next = eval {
>                 __PACKAGE__->new($marc, $row->biblionumber);
>             };
>diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm
>index f9eedf8..0ffe4f3 100644
>--- a/Koha/MetadataRecord/Authority.pm
>+++ b/Koha/MetadataRecord/Authority.pm
>@@ -150,52 +150,81 @@ sub authorized_heading {
> 
> =head2 get_all_authorities_iterator
> 
>-    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
>+    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
> 
> This will provide an iterator object that will, one by one, provide the
> Koha::MetadataRecord::Authority of each authority.
> 
> The iterator is a Koha::MetadataIterator object.
> 
>+Possible options are:
>+
>+=over 4
>+
>+=item C<slice>
>+
>+slice may be defined as a hash of two values: index and count. index
>+is the slice number to process and count is total number of slices. 
>+With this information the iterator returns just the given slice of
>+records instead of all.
>+
> =cut
> 
> sub get_all_authorities_iterator {
>+    my ($self, %options) = @_;
>+
>+    my ($slice_modulo, $slice_count);
>+    if ($options{slice}) {
>+        $slice_count = $options{slice}->{count};
>+        $slice_modulo = $options{slice}->{index};
>+        $slice_modulo = 0 if ($slice_modulo == $slice_count);
>+    }
>+
>     my $database = Koha::Database->new();
>     my $schema   = $database->schema();
>     my $rs =
>       $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
>         { columns => [qw/ authid authtypecode marcxml /] } );
>+
>     my $next_func = sub {
>-        my $row = $rs->next();
>-        return if !$row;
>-        my $authid       = $row->authid;
>-        my $authtypecode = $row->authtypecode;
>-        my $marcxml      = $row->marcxml;
>-
>-        my $record = eval {
>-            MARC::Record->new_from_xml(
>-                StripNonXmlChars($marcxml),
>-                'UTF-8',
>-                (
>-                    C4::Context->preference("marcflavour") eq "UNIMARC"
>-                    ? "UNIMARCAUTH"
>-                    : C4::Context->preference("marcflavour")
>-                )
>-            );
>-        };
>-        confess $@ if ($@);
>-        $record->encoding('UTF-8');
>-
>-        # I'm not sure why we don't use the authtypecode from the database,
>-        # but this is how the original code does it.
>-        require C4::AuthoritiesMarc;
>-        $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
>-
>-        my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } );
>-
>-        return $auth;
>-      };
>-      return Koha::MetadataIterator->new($next_func);
>+        while (1) {
>+            my $row = $rs->next();
>+            return if !$row;
>+
>+            # Check if authid matches our slice definition
>+            if (defined $slice_count) {
>+                next unless $row->authid % $slice_count == $slice_modulo;
>+            }
>+
>+            my $authid       = $row->authid;
>+            my $authtypecode = $row->authtypecode;
>+            my $marcxml      = $row->marcxml;
>+
>+            my $record = eval {
>+                MARC::Record->new_from_xml(
>+                    StripNonXmlChars($marcxml),
>+                    'UTF-8',
>+                    (
>+                        C4::Context->preference("marcflavour") eq "UNIMARC"
>+                        ? "UNIMARCAUTH"
>+                        : C4::Context->preference("marcflavour")
>+                    )
>+                );
>+            };
>+            confess $@ if ($@);
>+            $record->encoding('UTF-8');
>+
>+            # I'm not sure why we don't use the authtypecode from the database,
>+            # but this is how the original code does it.
>+            require C4::AuthoritiesMarc;
>+            $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
>+
>+            my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } );
>+
>+            return $auth;
>+        }
>+    };
>+    return Koha::MetadataIterator->new($next_func);
> }
> 
> 1;
>diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl
>index caa9a3d..cc2af11 100755
>--- a/misc/search_tools/rebuild_elastic_search.pl
>+++ b/misc/search_tools/rebuild_elastic_search.pl
>@@ -64,6 +64,14 @@ Only index the supplied biblionumber, mostly for testing purposes. May be
> repeated. This also applies to authorities via authid, so if you're using it,
> you probably only want to do one or the other at a time.
> 
>+=item B<-s|--slice>
>+
>+Only index a slice of the records. A slice is defined as two integers separated
>+by a comma. The first one is the slice to process and the second one is total
>+number of slices. e.g. --slice=1,3 processes every third record starting from
>+the first and --slice=2,3 processes every third record starting from the 
>+second.
>+
> =item B<-v|--verbose>
> 
> By default, this program only emits warnings and errors. This makes it talk
>@@ -94,21 +102,22 @@ use Pod::Usage;
> 
> my $verbose = 0;
> my $commit = 5000;
>-my ($delete, $help, $man);
>+my ($delete, $help, $man, $slice);
> my ($index_biblios, $index_authorities);
> my (@biblionumbers);
> 
> $|=1; # flushes output
> 
> GetOptions(
>-    'c|commit=i'       => \$commit,
>-    'd|delete'         => \$delete,
>+    'c|commit=i'    => \$commit,
>+    'd|delete'      => \$delete,
>     'a|authorities' => \$index_authorities,
>-    'b|biblios' => \$index_biblios,
>-    'bn|bnumber=i' => \@biblionumbers,
>-    'v|verbose+'       => \$verbose,
>-    'h|help'           => \$help,
>-    'man'              => \$man,
>+    'b|biblios'     => \$index_biblios,
>+    'bn|bnumber=i'  => \@biblionumbers,
>+    's|slice=s'     => \$slice,
>+    'v|verbose+'    => \$verbose,
>+    'h|help'        => \$help,
>+    'man'           => \$man,
> );
> 
> # Default is to do both
>@@ -121,6 +130,13 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
> 
> sanity_check();
> 
>+my %iterator_options;
>+if ($slice) {
>+    my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/;
>+    _log(1, "Processing slice $slice_index of $slice_count\n");
>+    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
>+}
>+
> my $next;
> if ($index_biblios) {
>     _log(1, "Indexing biblios\n");
>@@ -131,7 +147,7 @@ if ($index_biblios) {
>             return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
>         };
>     } else {
>-        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
>+        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
>         $next = sub {
>             $records->next();
>         }
>@@ -148,7 +164,7 @@ if ($index_authorities) {
>             return ($r, $a->record);
>         };
>     } else {
>-        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
>+        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
>         $next = sub {
>             $records->next();
>         }
>@@ -191,12 +207,12 @@ sub do_reindex {
>         push @id_buffer,     $id;
>         push @commit_buffer, $record;
>         if ( !( --$commit_count ) ) {
>-            _log( 1, "Committing $commit records..." );
>+            _log( 1, "Committing $commit records...\n" );
>             $indexer->update_index( \@id_buffer, \@commit_buffer );
>             $commit_count  = $commit;
>             @id_buffer     = ();
>             @commit_buffer = ();
>-            _log( 1, " done\n" );
>+            _log( 1, "Commit complete\n" );
>         }
>     }
> 
>@@ -222,5 +238,5 @@ sub sanity_check {
> sub _log {
>     my ($level, $msg) = @_;
> 
>-    print $msg if ($verbose >= $level);
>+    print "[$$] $msg" if ($verbose >= $level);
> }
>-- 
>2.7.4

Actions: View | Diff | Splinter Review

Attachments on bug 21872: 82582 | 82583 | 82584 | 82585 | 82586 | 82598 | 82599 | 82780 | 85119 | 85121 | 85975 | 85976 | 85977 | 85978 | 85979 | 86053 | 86054 | 86055 | 86056 | 86057 | 89109 | 89110 | 89111 | 89112 | 89113