From 523c697619153b758175474960e76337e3b5f906 Mon Sep 17 00:00:00 2001
From: Nick Clemens <nick@bywatersolutions.com>
Date: Thu, 2 May 2019 12:24:08 +0000
Subject: [PATCH] Bug 22831: Elasticsearch - add a maintenance script for
 checking DB vs index counts

This script uses Array::Utils and adds a dependency

To test:
 1 - Have Koha with ES running
 2 - Delete some records from ES
     curl -X DELETE "es:9200/koha_kohadev_biblios/data/5"
     curl -X DELETE "es:9200/koha_kohadev_authorities/data/5"
 3 - perl misc/maintenance/compare_es_to_db.pl
 4 - Note you are notified of problems in both indexes
 5 - perl misc/search_tools/rebuild_elastic_search.pl  -a
 6 - perl misc/maintenance/compare_es_to_db.pl
 7 - Note you are only notified about problems in biblios (assuming you don't have other issues)
 8 - perl misc/search_tools/rebuild_elastic_search.pl  -b
 9 - perl misc/maintenance/compare_es_to_db.pl
10 - Both counts match, no problems
---
 C4/Installer/PerlDependencies.pm     |  4 ++
 misc/maintenance/compare_es_to_db.pl | 76 ++++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 misc/maintenance/compare_es_to_db.pl

diff --git a/C4/Installer/PerlDependencies.pm b/C4/Installer/PerlDependencies.pm
index a62e633161..c7562277b0 100644
--- a/C4/Installer/PerlDependencies.pm
+++ b/C4/Installer/PerlDependencies.pm
@@ -917,6 +917,10 @@ our $PERL_DEPS = {
         'usage'    => 'Z39.50 responder',
         'required' => '0',
         'min_ver'  => '1.15',
+    'Array::Utils' => {
+        usage    => 'Elasticsearch integration',
+        required => '0',
+        min_ver  => '0.5',
     },
 };
 
diff --git a/misc/maintenance/compare_es_to_db.pl b/misc/maintenance/compare_es_to_db.pl
new file mode 100644
index 0000000000..fce73ad591
--- /dev/null
+++ b/misc/maintenance/compare_es_to_db.pl
@@ -0,0 +1,76 @@
+#! /usr/bin/perl
+#
+# This compares record counts from a Koha database to elastic search
+
+# Copyright 2019 ByWater Solutions
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+=head1 NAME
+
+compare_es_to_db.pl - compares record counts from a Koha database to Elasticsearch
+
+=head1 SYNOPSIS
+
+B<compare_es_to_db.pl>
+
+=cut
+
+use Modern::Perl;
+use Koha::Items;
+use Koha::SearchEngine::Elasticsearch;
+use Array::Utils qw( array_diff );
+
+use Koha::Biblios;
+use Koha::Authorities;
+
+foreach my $index ( ('biblios','authorities') ){
+    print "=================\n";
+    print "Checking $index\n";
+    my @db_records = $index eq 'biblios' ? Koha::Biblios->search()->get_column('biblionumber') : Koha::Authorities->search()->get_column('authid');
+
+    my $searcher = Koha::SearchEngine::Elasticsearch->new({ index => $index });
+    my $es = $searcher->get_elasticsearch();
+    my $count = $es->indices->stats( index => $searcher->get_elasticsearch_params->{index_name} )
+        ->{_all}{primaries}{docs}{count};
+    print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n";
+    # All done if they match
+    next if @db_records == $count;
+
+    # Otherwise, lets find all the ES ids
+    my $scroll = $es->scroll_helper(
+        index => $searcher->get_elasticsearch_params->{index_name},
+        size => 5000,
+        body => {
+            query => {
+                match_all => {}
+            }
+        },
+        scroll_in_qs => 1,
+    );
+
+    my @es_ids;
+
+    while (my $doc = $scroll->next ){
+        push @es_ids, $doc->{_id};
+    }
+
+    # And compare the arrays
+    my @diff = array_diff(@db_records, @es_ids );
+    foreach my $problem (@diff){
+        print "Record #$problem is not in both sources\n";
+    }
+}
-- 
2.11.0