From 7b090dd52682fbbf81e9b898686e2c9224eb7198 Mon Sep 17 00:00:00 2001 From: David Cook Date: Wed, 6 Jan 2021 03:01:55 +0000 Subject: [PATCH] Bug 27267: Add memory efficient Zebra indexing daemon This patch adds an alternate Zebra indexing daemon which is much more memory efficient than the current default daemon. (For instance, zebra_indexer.pl uses 6MB of RAM vs rebuild_zebra.pl which uses about 180MB of RAM out of the box while idle.) To test: 0. Apply patch 1. koha-indexer --stop kohadev 2. vi /etc/default/koha-common 3. Update ALTERNATE_INDEXER_DAEMON to be "/kohadevbox/koha/misc/zebra_indexer.pl" 4. koha-indexer --start kohadev 5. Go to http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=29 6. Change 245$a to "Gone" instead of "Gairm" and save the record 7. Go to http://localhost:8081/cgi-bin/koha/catalogue/search.pl?q=Gairm 8. Repeat Step 7 for 5+ seconds until your search returns no results 9. Go to http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=29 10. Change 245$a from "Gone" to "Gairm" 11. Repeat Step 7 for 5+ seconds until your search redirects you to http://localhost:8081/cgi-bin/koha/catalogue/detail.pl?biblionumber=29&found1=1 To confirm lower memory usage: 12. ps -efww | grep "zebra_indexer" 13. top -p 14. The top tool should show about 6.1MB usage. Signed-off-by: Martin Renvoize --- misc/zebra_indexer.pl | 108 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100755 misc/zebra_indexer.pl diff --git a/misc/zebra_indexer.pl b/misc/zebra_indexer.pl new file mode 100755 index 0000000000..45bd15e874 --- /dev/null +++ b/misc/zebra_indexer.pl @@ -0,0 +1,108 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use IPC::Open3; +use FindBin; + +while (1){ + run(); + sleep 5; +} + +sub run { + if ( _zebra_needs_indexing() ){ + _run_zebra(); + } +} + +#NOTE: This could also be done using a separate script but this is more self-contained +sub _zebra_needs_indexing { + my $pid; + if ( $pid = fork ){ + waitpid( $pid, 0 ); + my $child_exit_status = $? >> 8; + if ($child_exit_status == 0){ + return 1; + } + else { + return; + } + } + elsif ( ! defined $pid ){ + warn "Failed to fork!"; + exit 1; + } + elsif ( $pid == 0 ){ + my $count = _get_count(); + if ( $count && $count > 0 ){ + exit 0; + } + else { + exit 1; + } + } +} + +sub _run_zebra { + my $zebra_indexer = "$FindBin::Bin/migration_tools/rebuild_zebra.pl"; + my @command = ($zebra_indexer,"-b","-a","-z"); + + my $pid = open3(my $child_in, '>&STDOUT', '>&STDERR', @command); + waitpid($pid,0); +} + +sub _get_count { + my $count; + my $dbh = _get_dbh(); + if ($dbh){ + my $sql = "SELECT COUNT(*) AS count FROM zebraqueue WHERE done = 0;"; + my $sth = $dbh->prepare($sql); + if ($sth){ + $sth->execute(); + $count = $sth->fetchrow_arrayref->[0]; + } + } + return $count; +} + +sub _get_dbh { + my $conf = _get_db_config($ENV{KOHA_CONF}); + my $dsn = "dbi:$conf->{db_driver}:database=$conf->{db_name};host=$conf->{db_host};port=$conf->{db_port}".($conf->{tls_options}? $conf->{tls_options} : ""); + require DBI; + my $dbh = DBI->connect("$dsn","$conf->{user}","$conf->{pass}"); + return $dbh; +} + +sub _get_db_config { + my ($kohaconf_filename) = @_; + my $db_config = {}; + require XML::LibXML; + my $doc = XML::LibXML->load_xml(location => $kohaconf_filename); + my $root = $doc->documentElement; + my @nodelist = $root->findnodes("./config"); + my $config = $nodelist[0]; + $db_config->{db_driver} = _get_value($config,"db_scheme"); + $db_config->{db_name} = _get_value($config,"database"); + $db_config->{db_host} = _get_value($config,"hostname"); + $db_config->{db_port} = _get_value($config,"port") || ''; + $db_config->{user} = _get_value($config,"user"); + $db_config->{pass} = _get_value($config,"pass"); + my $tls = _get_value($config,"tls"); + if ($tls && $tls eq 'yes'){ + my $ca = _get_value($config,"ca"); + my $cert = _get_value($config,"cert"); + my $key = _get_value($config,"key"); + $db_config->{tls_options} = ";mysql_ssl=1;mysql_ssl_client_key=".$key.";mysql_ssl_client_cert=".$cert.";mysql_ssl_ca_file=".$ca; + } + return $db_config; +} + +sub _get_value { + my ($parent,$tag) = @_; + my $node = $parent->findnodes("./$tag"); + if ($node){ + return $node->to_literal(); + } +} -- 2.20.1