Lines 30-41
B<compare_es_to_db.pl>
Link Here
|
30 |
=cut |
30 |
=cut |
31 |
|
31 |
|
32 |
use Modern::Perl; |
32 |
use Modern::Perl; |
33 |
use Koha::Items; |
|
|
34 |
use Koha::SearchEngine::Elasticsearch; |
35 |
use Array::Utils qw( array_diff ); |
33 |
use Array::Utils qw( array_diff ); |
36 |
|
34 |
|
37 |
use Koha::Biblios; |
35 |
use C4::Context; |
|
|
36 |
|
38 |
use Koha::Authorities; |
37 |
use Koha::Authorities; |
|
|
38 |
use Koha::Biblios; |
39 |
use Koha::Items; |
40 |
use Koha::SearchEngine::Elasticsearch; |
39 |
|
41 |
|
40 |
foreach my $index ( ('biblios','authorities') ){ |
42 |
foreach my $index ( ('biblios','authorities') ){ |
41 |
print "=================\n"; |
43 |
print "=================\n"; |
Lines 48-54
foreach my $index ( ('biblios','authorities') ){
Link Here
|
48 |
->{_all}{primaries}{docs}{count}; |
50 |
->{_all}{primaries}{docs}{count}; |
49 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
51 |
print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n"; |
50 |
|
52 |
|
51 |
# Otherwise, lets find all the ES ids |
53 |
# Now we get all the ids from Elasticsearch |
|
|
54 |
# The scroll lets us iterate through, it fetches chunks of 'size' as we move through |
52 |
my $scroll = $es->scroll_helper( |
55 |
my $scroll = $es->scroll_helper( |
53 |
index => $searcher->get_elasticsearch_params->{index_name}, |
56 |
index => $searcher->get_elasticsearch_params->{index_name}, |
54 |
size => 5000, |
57 |
size => 5000, |
Lines 63-82
foreach my $index ( ('biblios','authorities') ){
Link Here
|
63 |
|
66 |
|
64 |
my @es_ids; |
67 |
my @es_ids; |
65 |
|
68 |
|
|
|
69 |
# Here is where we actually iterate through |
70 |
# Fetching each record, pushing the id into the array |
66 |
my $i = 1; |
71 |
my $i = 1; |
67 |
print "Fetching Elasticsearch records ids"; |
72 |
print "Fetching Elasticsearch records ids"; |
68 |
while (my $doc = $scroll->next ){ |
73 |
while (my $doc = $scroll->next ){ |
69 |
print "." if !($i % 500); |
74 |
print "." if !($i % 500); |
70 |
print "\nFetching next 5000" if !($i % 5000); |
75 |
print "\n$i records retrieved" if !($i % 5000); |
71 |
push @es_ids, $doc->{_id}; |
76 |
push @es_ids, $doc->{_id}; |
72 |
$i++; |
77 |
$i++; |
73 |
} |
78 |
} |
74 |
print "\nComparing arrays, this may take a while\n"; |
79 |
print "\nComparing arrays, this may take a while\n"; |
75 |
|
80 |
|
76 |
# And compare the arrays |
81 |
# And compare the arrays |
|
|
82 |
# array_diff returns only a list of values which are not common to both arrays |
77 |
my @diff = array_diff(@db_records, @es_ids ); |
83 |
my @diff = array_diff(@db_records, @es_ids ); |
|
|
84 |
|
78 |
print "All records match\n" unless @diff; |
85 |
print "All records match\n" unless @diff; |
|
|
86 |
|
87 |
# Fetch values for providing record links |
88 |
my $es_params = $searcher->get_elasticsearch_params; |
89 |
my $es_base = "$es_params->{nodes}[0]/$es_params->{index_name}"; |
90 |
my $opac_base = C4::Context->preference('OPACBaseURL'); |
91 |
|
92 |
|
79 |
foreach my $problem (@diff){ |
93 |
foreach my $problem (@diff){ |
80 |
print "Record #$problem is not in both sources\n"; |
94 |
if ( (grep /^$problem$/, @db_records) ){ |
|
|
95 |
print "Record $problem exists in Koha but not ES\n"; |
96 |
if ( $index eq 'biblios' ) { |
97 |
print " Visit here to see record: $opac_base/cgi-bin/koha/opac-detail.pl?biblionumber=$problem\n"; |
98 |
} elsif ( $index eq 'authorities' ) { |
99 |
print " Visit here to see record: $opac_base/cgi-bin/koha/opac-authoritiesdetail.pl?authid=$problem\n"; |
100 |
} |
101 |
} else { |
102 |
print "Record $problem exists in ES but not Koha\n"; |
103 |
print " Enter this command to view record: curl $es_base/data/$problem?pretty=true\n"; |
104 |
} |
81 |
} |
105 |
} |
82 |
} |
106 |
} |
83 |
- |
|
|