Line 0
Link Here
|
0 |
- |
1 |
#! /usr/bin/perl |
|
|
2 |
# |
3 |
# This compares record counts from a Koha database to Elasticsearch |
4 |
|
5 |
# Copyright 2019 ByWater Solutions |
6 |
# |
7 |
# This file is part of Koha. |
8 |
# |
9 |
# Koha is free software; you can redistribute it and/or modify it under the |
10 |
# terms of the GNU General Public License as published by the Free Software |
11 |
# Foundation; either version 3 of the License, or (at your option) any later |
12 |
# version. |
13 |
# |
14 |
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY |
15 |
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
16 |
# A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
17 |
# |
18 |
# You should have received a copy of the GNU General Public License along |
19 |
# with Koha; if not, write to the Free Software Foundation, Inc., |
20 |
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
21 |
|
22 |
=head1 NAME |
23 |
|
24 |
compare_es_to_db.pl - compares record counts from a Koha database to Elasticsearch |
25 |
|
26 |
=head1 SYNOPSIS |
27 |
|
28 |
B<compare_es_to_db.pl> |
29 |
|
30 |
=cut |
31 |
|
32 |
use Modern::Perl; |
33 |
use Koha::Items; |
34 |
use Koha::SearchEngine::Elasticsearch; |
35 |
use Array::Utils qw( array_diff ); |
36 |
|
37 |
use Koha::Biblios; |
38 |
use Koha::Authorities; |
39 |
|
40 |
foreach my $index ( ('biblios','authorities') ){ |
41 |
print "=================\n"; |
42 |
print "Checking $index\n"; |
43 |
my @db_records = $index eq 'biblios' ? Koha::Biblios->search()->get_column('biblionumber') : Koha::Authorities->search()->get_column('authid'); |
44 |
|
45 |
my $searcher = Koha::SearchEngine::Elasticsearch->new({ index => $index }); |
46 |
my $es = $searcher->get_elasticsearch(); |
47 |
my $count = $es->indices->stats( index => $searcher->get_elasticsearch_params->{index_name} ) |
48 |
->{_all}{primaries}{docs}{count}; |
49 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
50 |
|
51 |
# Otherwise, lets find all the ES ids |
52 |
my $scroll = $es->scroll_helper( |
53 |
index => $searcher->get_elasticsearch_params->{index_name}, |
54 |
size => 5000, |
55 |
body => { |
56 |
query => { |
57 |
match_all => {} |
58 |
}, |
59 |
stored_fields => [] |
60 |
}, |
61 |
scroll_in_qs => 1, |
62 |
); |
63 |
|
64 |
my @es_ids; |
65 |
|
66 |
my $i = 1; |
67 |
print "Fetching Elasticsearch records ids"; |
68 |
while (my $doc = $scroll->next ){ |
69 |
print "." if !($i % 500); |
70 |
print "\nFetching next 5000" if !($i % 5000); |
71 |
push @es_ids, $doc->{_id}; |
72 |
$i++; |
73 |
} |
74 |
print "\nComparing arrays, this may take a while\n"; |
75 |
|
76 |
# And compare the arrays |
77 |
my @diff = array_diff(@db_records, @es_ids ); |
78 |
print "All records match\n" unless @diff; |
79 |
foreach my $problem (@diff){ |
80 |
print "Record #$problem is not in both sources\n"; |
81 |
} |
82 |
} |