Attachment 120329 Details for Bug 28268 – Bug 28268: Improve memory usage when indexing authorities in Elasticsearch

[patch] Bug 28268: Improve memory usage when indexing authorities in Elasticsearch

Bug-28268-Improve-memory-usage-when-indexing-autho.patch (text/plain), 2.62 KB, created by Ere Maijala on 2021-04-30 08:45:16 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2021-04-30 08:45:16 UTC

Size: 2.62 KB

patch

obsolete

>From a8fc7a47cbcde6017c619a3f1f77c57702455f33 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Fri, 30 Apr 2021 10:00:23 +0300
>Subject: [PATCH] Bug 28268: Improve memory usage when indexing authorities in
> Elasticsearch
>
>Retrieves the complete records one by one to avoid huge memory usage.
>
>Note that this removes the call to GuessAuthTypeCode, but it is done later in Koha::SearchEngine::Elasticsearch::marc_records_to_documents (and was never done if you asked to index a single record with --authid parameter).
>
>Test plan:
>1. Apply patch
>2. Reindex authorities: perl misc/search_tools/rebuild_elasticsearch.pl -a -d -v
>3. Check that indexing completed successfully and results are correct.
>---
> Koha/MetadataRecord/Authority.pm | 31 ++++++-------------------------
> 1 file changed, 6 insertions(+), 25 deletions(-)
>
>diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm
>index 47f2172976..3c0bfcd061 100644
>--- a/Koha/MetadataRecord/Authority.pm
>+++ b/Koha/MetadataRecord/Authority.pm
>@@ -191,7 +191,7 @@ sub get_all_authorities_iterator {
>         };
>     }
> 
>-    my $search_options->{columns} = [qw/ authid authtypecode marcxml /];
>+    my $search_options->{columns} = [qw/ authid /];
>     if ($options{desc}) {
>         $search_options->{order_by} = { -desc => 'authid' };
>     }
>@@ -205,31 +205,12 @@ sub get_all_authorities_iterator {
>     my $next_func = sub {
>         my $row = $rs->next();
>         return if !$row;
>-        my $authid       = $row->authid;
>-        my $authtypecode = $row->authtypecode;
>-        my $marcxml      = $row->marcxml;
>-
>-        my $record = eval {
>-            MARC::Record->new_from_xml(
>-                StripNonXmlChars($marcxml),
>-                'UTF-8',
>-                (
>-                    C4::Context->preference("marcflavour") eq "UNIMARC"
>-                    ? "UNIMARCAUTH"
>-                    : C4::Context->preference("marcflavour")
>-                )
>-            );
>-        };
>-        confess "$@" if ($@);
>-        $record->encoding('UTF-8');
>-
>-        # I'm not sure why we don't use the authtypecode from the database,
>-        # but this is how the original code does it.
>-        require C4::AuthoritiesMarc;
>-        $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
>-
>-        my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } );
> 
>+        my $auth = __PACKAGE__->get_from_authid($row->authid);
>+        if (!$auth) {
>+            confess "$@" if ($@);
>+            return;
>+        }
>         return $auth;
>       };
>       return Koha::MetadataIterator->new($next_func);
>-- 
>2.25.1

Actions: View | Diff | Splinter Review

Attachments on bug 28268: 120329 | 120330 | 120424 | 120865 | 120866