|
Line 0
Link Here
|
| 0 |
- |
1 |
#! /usr/bin/perl |
|
|
2 |
# |
| 3 |
# This compares record counts from a Koha database to Elasticsearch |
| 4 |
|
| 5 |
# Copyright 2019 ByWater Solutions |
| 6 |
# |
| 7 |
# This file is part of Koha. |
| 8 |
# |
| 9 |
# Koha is free software; you can redistribute it and/or modify it under the |
| 10 |
# terms of the GNU General Public License as published by the Free Software |
| 11 |
# Foundation; either version 3 of the License, or (at your option) any later |
| 12 |
# version. |
| 13 |
# |
| 14 |
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY |
| 15 |
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
| 16 |
# A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
| 17 |
# |
| 18 |
# You should have received a copy of the GNU General Public License along |
| 19 |
# with Koha; if not, write to the Free Software Foundation, Inc., |
| 20 |
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 21 |
|
| 22 |
=head1 NAME |
| 23 |
|
| 24 |
compare_es_to_db.pl - compares record counts from a Koha database to Elasticsearch |
| 25 |
|
| 26 |
=head1 SYNOPSIS |
| 27 |
|
| 28 |
B<compare_es_to_db.pl> |
| 29 |
|
| 30 |
=cut |
| 31 |
|
| 32 |
use Modern::Perl; |
| 33 |
use Koha::Items; |
| 34 |
use Koha::SearchEngine::Elasticsearch; |
| 35 |
use Array::Utils qw( array_diff ); |
| 36 |
|
| 37 |
use Koha::Biblios; |
| 38 |
use Koha::Authorities; |
| 39 |
|
| 40 |
foreach my $index ( ('biblios','authorities') ){ |
| 41 |
print "=================\n"; |
| 42 |
print "Checking $index\n"; |
| 43 |
my @db_records = $index eq 'biblios' ? Koha::Biblios->search()->get_column('biblionumber') : Koha::Authorities->search()->get_column('authid'); |
| 44 |
|
| 45 |
my $searcher = Koha::SearchEngine::Elasticsearch->new({ index => $index }); |
| 46 |
my $es = $searcher->get_elasticsearch(); |
| 47 |
my $count = $es->indices->stats( index => $searcher->get_elasticsearch_params->{index_name} ) |
| 48 |
->{_all}{primaries}{docs}{count}; |
| 49 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
| 50 |
|
| 51 |
# Otherwise, lets find all the ES ids |
| 52 |
my $scroll = $es->scroll_helper( |
| 53 |
index => $searcher->get_elasticsearch_params->{index_name}, |
| 54 |
size => 5000, |
| 55 |
body => { |
| 56 |
query => { |
| 57 |
match_all => {} |
| 58 |
}, |
| 59 |
stored_fields => [] |
| 60 |
}, |
| 61 |
scroll_in_qs => 1, |
| 62 |
); |
| 63 |
|
| 64 |
my @es_ids; |
| 65 |
|
| 66 |
my $i = 1; |
| 67 |
print "Fetching Elasticsearch records ids"; |
| 68 |
while (my $doc = $scroll->next ){ |
| 69 |
print "." if !($i % 500); |
| 70 |
print "\nFetching next 5000" if !($i % 5000); |
| 71 |
push @es_ids, $doc->{_id}; |
| 72 |
$i++; |
| 73 |
} |
| 74 |
print "\nComparing arrays, this may take a while\n"; |
| 75 |
|
| 76 |
# And compare the arrays |
| 77 |
my @diff = array_diff(@db_records, @es_ids ); |
| 78 |
print "All records match\n" unless @diff; |
| 79 |
foreach my $problem (@diff){ |
| 80 |
print "Record #$problem is not in both sources\n"; |
| 81 |
} |
| 82 |
} |