View | Details | Raw Unified | Return to bug 22831
Collapse All | Expand All

(-)a/misc/maintenance/compare_es_to_db.pl (-7 / +30 lines)
Lines 30-41 B<compare_es_to_db.pl> Link Here
30
=cut
30
=cut
31
31
32
use Modern::Perl;
32
use Modern::Perl;
33
use Koha::Items;
34
use Koha::SearchEngine::Elasticsearch;
35
use Array::Utils qw( array_diff );
33
use Array::Utils qw( array_diff );
36
34
37
use Koha::Biblios;
35
use C4::Context;
36
38
use Koha::Authorities;
37
use Koha::Authorities;
38
use Koha::Biblios;
39
use Koha::Items;
40
use Koha::SearchEngine::Elasticsearch;
39
41
40
foreach my $index ( ('biblios','authorities') ){
42
foreach my $index ( ('biblios','authorities') ){
41
    print "=================\n";
43
    print "=================\n";
Lines 48-54 foreach my $index ( ('biblios','authorities') ){ Link Here
48
        ->{_all}{primaries}{docs}{count};
50
        ->{_all}{primaries}{docs}{count};
49
    print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n";
51
    print "Count in db for $index is " . scalar @db_records . ", count in Elasticsearch is $count\n";
50
52
51
    # Otherwise, lets find all the ES ids
53
    # Now we get all the ids from Elasticsearch
54
    # The scroll lets us iterate through, it fetches chunks of 'size' as we move through
52
    my $scroll = $es->scroll_helper(
55
    my $scroll = $es->scroll_helper(
53
        index => $searcher->get_elasticsearch_params->{index_name},
56
        index => $searcher->get_elasticsearch_params->{index_name},
54
        size => 5000,
57
        size => 5000,
Lines 63-82 foreach my $index ( ('biblios','authorities') ){ Link Here
63
66
64
    my @es_ids;
67
    my @es_ids;
65
68
69
    # Here is where we actually iterate through
70
    # Fetching each record, pushing the id into the array
66
    my $i = 1;
71
    my $i = 1;
67
    print "Fetching Elasticsearch records ids";
72
    print "Fetching Elasticsearch records ids";
68
    while (my $doc = $scroll->next ){
73
    while (my $doc = $scroll->next ){
69
        print "." if !($i % 500);
74
        print "." if !($i % 500);
70
        print "\nFetching next 5000" if !($i % 5000);
75
        print "\n$i records retrieved" if !($i % 5000);
71
        push @es_ids, $doc->{_id};
76
        push @es_ids, $doc->{_id};
72
        $i++;
77
        $i++;
73
    }
78
    }
74
    print "\nComparing arrays, this may take a while\n";
79
    print "\nComparing arrays, this may take a while\n";
75
80
76
    # And compare the arrays
81
    # And compare the arrays
82
    # array_diff returns only a list of values which are not common to both arrays
77
    my @diff = array_diff(@db_records, @es_ids );
83
    my @diff = array_diff(@db_records, @es_ids );
84
78
    print "All records match\n" unless @diff;
85
    print "All records match\n" unless @diff;
86
87
    # Fetch values for providing record links
88
    my $es_params = $searcher->get_elasticsearch_params;
89
    my $es_base   = "$es_params->{nodes}[0]/$es_params->{index_name}";
90
    my $opac_base = C4::Context->preference('OPACBaseURL');
91
92
79
    foreach my $problem (@diff){
93
    foreach my $problem (@diff){
80
        print "Record #$problem is not in both sources\n";
94
        if ( (grep /^$problem$/, @db_records) ){
95
            print "Record $problem exists in Koha but not ES\n";
96
            if ( $index eq 'biblios' ) {
97
                print "  Visit here to see record: $opac_base/cgi-bin/koha/opac-detail.pl?biblionumber=$problem\n";
98
            } elsif ( $index eq 'authorities' ) {
99
                print "  Visit here to see record: $opac_base/cgi-bin/koha/opac-authoritiesdetail.pl?authid=$problem\n";
100
            }
101
        } else {
102
            print "Record $problem exists in ES but not Koha\n";
103
            print "  Enter this command to view record: curl $es_base/data/$problem?pretty=true\n";
104
        }
81
    }
105
    }
82
}
106
}
83
- 

Return to bug 22831