|
Lines 30-41
B<compare_es_to_db.pl>
Link Here
|
| 30 |
=cut |
30 |
=cut |
| 31 |
|
31 |
|
| 32 |
use Modern::Perl; |
32 |
use Modern::Perl; |
| 33 |
use Koha::Items; |
|
|
| 34 |
use Koha::SearchEngine::Elasticsearch; |
| 35 |
use Array::Utils qw( array_diff ); |
33 |
use Array::Utils qw( array_diff ); |
| 36 |
|
34 |
|
| 37 |
use Koha::Biblios; |
35 |
use C4::Context; |
|
|
36 |
|
| 38 |
use Koha::Authorities; |
37 |
use Koha::Authorities; |
|
|
38 |
use Koha::Biblios; |
| 39 |
use Koha::Items; |
| 40 |
use Koha::SearchEngine::Elasticsearch; |
| 39 |
|
41 |
|
| 40 |
foreach my $index ( ('biblios','authorities') ){ |
42 |
foreach my $index ( ('biblios','authorities') ){ |
| 41 |
print "=================\n"; |
43 |
print "=================\n"; |
|
Lines 48-54
foreach my $index ( ('biblios','authorities') ){
Link Here
|
| 48 |
->{_all}{primaries}{docs}{count}; |
50 |
->{_all}{primaries}{docs}{count}; |
| 49 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
51 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
| 50 |
|
52 |
|
| 51 |
# Otherwise, lets find all the ES ids |
53 |
# Now we get all the ids from Elasticsearch |
|
|
54 |
# The scroll lets us iterate through, it fetches chunks of 'size' as we move through |
| 52 |
my $scroll = $es->scroll_helper( |
55 |
my $scroll = $es->scroll_helper( |
| 53 |
index => $searcher->get_elasticsearch_params->{index_name}, |
56 |
index => $searcher->get_elasticsearch_params->{index_name}, |
| 54 |
size => 5000, |
57 |
size => 5000, |
|
Lines 63-82
foreach my $index ( ('biblios','authorities') ){
Link Here
|
| 63 |
|
66 |
|
| 64 |
my @es_ids; |
67 |
my @es_ids; |
| 65 |
|
68 |
|
|
|
69 |
# Here is where we actually iterate through |
| 70 |
# Fetching each record, pushing the id into the array |
| 66 |
my $i = 1; |
71 |
my $i = 1; |
| 67 |
print "Fetching Elasticsearch records ids"; |
72 |
print "Fetching Elasticsearch records ids"; |
| 68 |
while (my $doc = $scroll->next ){ |
73 |
while (my $doc = $scroll->next ){ |
| 69 |
print "." if !($i % 500); |
74 |
print "." if !($i % 500); |
| 70 |
print "\nFetching next 5000" if !($i % 5000); |
75 |
print "\n$i records retrieved" if !($i % 5000); |
| 71 |
push @es_ids, $doc->{_id}; |
76 |
push @es_ids, $doc->{_id}; |
| 72 |
$i++; |
77 |
$i++; |
| 73 |
} |
78 |
} |
| 74 |
print "\nComparing arrays, this may take a while\n"; |
79 |
print "\nComparing arrays, this may take a while\n"; |
| 75 |
|
80 |
|
| 76 |
# And compare the arrays |
81 |
# And compare the arrays |
|
|
82 |
# array_diff returns only a list of values which are not common to both arrays |
| 77 |
my @diff = array_diff(@db_records, @es_ids ); |
83 |
my @diff = array_diff(@db_records, @es_ids ); |
|
|
84 |
|
| 78 |
print "All records match\n" unless @diff; |
85 |
print "All records match\n" unless @diff; |
|
|
86 |
|
| 87 |
# Fetch values for providing record links |
| 88 |
my $es_params = $searcher->get_elasticsearch_params; |
| 89 |
my $es_base = "$es_params->{nodes}[0]/$es_params->{index_name}"; |
| 90 |
my $opac_base = C4::Context->preference('OPACBaseURL'); |
| 91 |
|
| 92 |
|
| 79 |
foreach my $problem (@diff){ |
93 |
foreach my $problem (@diff){ |
| 80 |
print "Record #$problem is not in both sources\n"; |
94 |
if ( (grep /^$problem$/, @db_records) ){ |
|
|
95 |
print "Record $problem exists in Koha but not ES\n"; |
| 96 |
if ( $index eq 'biblios' ) { |
| 97 |
print " Visit here to see record: $opac_base/cgi-bin/koha/opac-detail.pl?biblionumber=$problem\n"; |
| 98 |
} elsif ( $index eq 'authorities' ) { |
| 99 |
print " Visit here to see record: $opac_base/cgi-bin/koha/opac-authoritiesdetail.pl?authid=$problem\n"; |
| 100 |
} |
| 101 |
} else { |
| 102 |
print "Record $problem exists in ES but not Koha\n"; |
| 103 |
print " Enter this command to view record: curl $es_base/data/$problem?pretty=true\n"; |
| 104 |
} |
| 81 |
} |
105 |
} |
| 82 |
} |
106 |
} |
| 83 |
- |
|
|