Attachment 117437 Details for Bug 27267 – Bug 27267: Add memory efficient Zebra indexing daemon

[patch] Bug 27267: Add memory efficient Zebra indexing daemon

Bug-27267-Add-memory-efficient-Zebra-indexing-daem.patch (text/plain), 4.65 KB, created by Martin Renvoize (ashimema) on 2021-03-01 11:18:59 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Martin Renvoize (ashimema)

Created: 2021-03-01 11:18:59 UTC

Size: 4.65 KB

patch

obsolete

>From 7b090dd52682fbbf81e9b898686e2c9224eb7198 Mon Sep 17 00:00:00 2001
>From: David Cook <dcook@prosentient.com.au>
>Date: Wed, 6 Jan 2021 03:01:55 +0000
>Subject: [PATCH] Bug 27267: Add memory efficient Zebra indexing daemon
>
>This patch adds an alternate Zebra indexing daemon which
>is much more memory efficient than the current default daemon.
>
>(For instance, zebra_indexer.pl uses 6MB of RAM vs rebuild_zebra.pl
> which uses about 180MB of RAM out of the box while idle.)
>
>To test:
>0. Apply patch
>1. koha-indexer --stop kohadev
>2. vi /etc/default/koha-common
>3. Update ALTERNATE_INDEXER_DAEMON to be "/kohadevbox/koha/misc/zebra_indexer.pl"
>4. koha-indexer --start kohadev
>5. Go to http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=29
>6. Change 245$a to "Gone" instead of "Gairm" and save the record
>7. Go to http://localhost:8081/cgi-bin/koha/catalogue/search.pl?q=Gairm
>8. Repeat Step 7 for 5+ seconds until your search returns no results
>9. Go to http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=29
>10. Change 245$a from "Gone" to "Gairm"
>11. Repeat Step 7 for 5+ seconds until your search redirects you to
>http://localhost:8081/cgi-bin/koha/catalogue/detail.pl?biblionumber=29&found1=1
>
>To confirm lower memory usage:
>12. ps -efww | grep "zebra_indexer"
>13. top -p <PID of zebra_indexer.pl process>
>14. The top tool should show about 6.1MB usage.
>
>Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
>---
> misc/zebra_indexer.pl | 108 ++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 108 insertions(+)
> create mode 100755 misc/zebra_indexer.pl
>
>diff --git a/misc/zebra_indexer.pl b/misc/zebra_indexer.pl
>new file mode 100755
>index 0000000000..45bd15e874
>--- /dev/null
>+++ b/misc/zebra_indexer.pl
>@@ -0,0 +1,108 @@
>+#!/usr/bin/perl
>+
>+use strict;
>+use warnings;
>+
>+use IPC::Open3;
>+use FindBin;
>+
>+while (1){
>+    run();
>+    sleep 5;
>+}
>+
>+sub run {
>+    if ( _zebra_needs_indexing() ){
>+        _run_zebra();
>+    }
>+}
>+
>+#NOTE: This could also be done using a separate script but this is more self-contained
>+sub _zebra_needs_indexing {
>+    my $pid;
>+    if ( $pid = fork ){
>+        waitpid( $pid, 0 );
>+        my $child_exit_status = $? >> 8;
>+        if ($child_exit_status == 0){
>+            return 1;
>+        }
>+        else {
>+            return;
>+        }
>+    }
>+    elsif ( ! defined $pid ){
>+        warn "Failed to fork!";
>+        exit 1;
>+    }
>+    elsif ( $pid == 0 ){
>+        my $count = _get_count();
>+        if ( $count && $count > 0 ){
>+            exit 0;
>+        }
>+        else {
>+            exit 1;
>+        }
>+    }
>+}
>+
>+sub _run_zebra {
>+    my $zebra_indexer = "$FindBin::Bin/migration_tools/rebuild_zebra.pl";
>+    my @command = ($zebra_indexer,"-b","-a","-z");
>+
>+    my $pid = open3(my $child_in, '>&STDOUT', '>&STDERR', @command);
>+    waitpid($pid,0);
>+}
>+
>+sub _get_count {
>+    my $count;
>+    my $dbh = _get_dbh();
>+    if ($dbh){
>+        my $sql = "SELECT COUNT(*) AS count FROM zebraqueue WHERE done = 0;";
>+        my $sth = $dbh->prepare($sql);
>+        if ($sth){
>+            $sth->execute();
>+            $count = $sth->fetchrow_arrayref->[0];
>+        }
>+    }
>+    return $count;
>+}
>+
>+sub _get_dbh {
>+    my $conf = _get_db_config($ENV{KOHA_CONF});
>+    my $dsn = "dbi:$conf->{db_driver}:database=$conf->{db_name};host=$conf->{db_host};port=$conf->{db_port}".($conf->{tls_options}? $conf->{tls_options} : "");
>+    require DBI;
>+    my $dbh = DBI->connect("$dsn","$conf->{user}","$conf->{pass}");
>+    return $dbh;
>+}
>+
>+sub _get_db_config {
>+    my ($kohaconf_filename) = @_;
>+    my $db_config = {};
>+    require XML::LibXML;
>+    my $doc = XML::LibXML->load_xml(location => $kohaconf_filename);
>+    my $root = $doc->documentElement;
>+    my @nodelist = $root->findnodes("./config");
>+    my $config = $nodelist[0];
>+    $db_config->{db_driver} = _get_value($config,"db_scheme");
>+    $db_config->{db_name} = _get_value($config,"database");
>+    $db_config->{db_host} = _get_value($config,"hostname");
>+    $db_config->{db_port} = _get_value($config,"port") || '';
>+    $db_config->{user} = _get_value($config,"user");
>+    $db_config->{pass} = _get_value($config,"pass");
>+    my $tls = _get_value($config,"tls");
>+    if ($tls && $tls eq 'yes'){
>+        my $ca = _get_value($config,"ca");
>+        my $cert = _get_value($config,"cert");
>+        my $key = _get_value($config,"key");
>+        $db_config->{tls_options} = ";mysql_ssl=1;mysql_ssl_client_key=".$key.";mysql_ssl_client_cert=".$cert.";mysql_ssl_ca_file=".$ca;
>+    }
>+    return $db_config;
>+}
>+
>+sub _get_value {
>+    my ($parent,$tag) = @_;
>+    my $node = $parent->findnodes("./$tag");
>+    if ($node){
>+        return $node->to_literal();
>+    }
>+}
>-- 
>2.20.1

Actions: View | Diff | Splinter Review

Attachments on bug 27267: 114884 | 117437