View | Details | Raw Unified | Return to bug 18948
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-34 / +99 lines)
Lines 41-46 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here
41
41
42
=head1 FUNCTIONS
42
=head1 FUNCTIONS
43
43
44
=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name});
45
46
$index_name should be biblios or authorities
47
call our SUPER class and then set ES parameters
48
49
=cut
50
51
sub new {
52
    my $class = shift @_;
53
    my $self = $class->SUPER::new(@_);
54
    my $params  = $self->get_elasticsearch_params();
55
    $self->store(
56
        Catmandu::Store::ElasticSearch->new(
57
            %$params,
58
            index_settings => $self->get_elasticsearch_settings(),
59
            index_mappings => $self->get_elasticsearch_mappings(),
60
        )
61
    );
62
    return $self;
63
}
64
44
=head2 $indexer->update_index($biblionums, $records);
65
=head2 $indexer->update_index($biblionums, $records);
45
66
46
C<$biblionums> is an arrayref containing the biblionumbers for the records.
67
C<$biblionums> is an arrayref containing the biblionumbers for the records.
Lines 67-84 sub update_index { Link Here
67
    }
88
    }
68
89
69
    my $from    = $self->_convert_marc_to_json($records);
90
    my $from    = $self->_convert_marc_to_json($records);
70
    if ( !$self->store ) {
71
        my $params  = $self->get_elasticsearch_params();
72
        $self->store(
73
            Catmandu::Store::ElasticSearch->new(
74
                %$params,
75
                index_settings => $self->get_elasticsearch_settings(),
76
                index_mappings => $self->get_elasticsearch_mappings(),
77
            )
78
        );
79
    }
80
91
81
    #print Data::Dumper::Dumper( $from->to_array );
92
    print Data::Dumper::Dumper( $from->to_array );
82
    $self->store->bag->add_many($from);
93
    $self->store->bag->add_many($from);
83
    $self->store->bag->commit;
94
    $self->store->bag->commit;
84
    return 1;
95
    return 1;
Lines 111-126 C<$biblionums> is an arrayref of biblionumbers to delete from the index. Link Here
111
sub delete_index {
122
sub delete_index {
112
    my ($self, $biblionums) = @_;
123
    my ($self, $biblionums) = @_;
113
124
114
    if ( !$self->store ) {
115
        my $params  = $self->get_elasticsearch_params();
116
        $self->store(
117
            Catmandu::Store::ElasticSearch->new(
118
                %$params,
119
                index_settings => $self->get_elasticsearch_settings(),
120
                index_mappings => $self->get_elasticsearch_mappings(),
121
            )
122
        );
123
    }
124
    $self->store->bag->delete($_) foreach @$biblionums;
125
    $self->store->bag->delete($_) foreach @$biblionums;
125
    $self->store->bag->commit;
126
    $self->store->bag->commit;
126
}
127
}
Lines 149-168 after this will recreate it again. Link Here
149
sub drop_index {
150
sub drop_index {
150
    my ($self) = @_;
151
    my ($self) = @_;
151
152
152
    if (!$self->store) {
153
        # If this index doesn't exist, this will create it. Then it'll be
154
        # deleted. That's not the end of the world however.
155
        my $params  = $self->get_elasticsearch_params();
156
        $self->store(
157
            Catmandu::Store::ElasticSearch->new(
158
                %$params,
159
                index_settings => $self->get_elasticsearch_settings(),
160
                index_mappings => $self->get_elasticsearch_mappings(),
161
            )
162
        );
163
    }
164
    $self->store->drop();
153
    $self->store->drop();
165
    $self->store(undef);
154
    my $params  = $self->get_elasticsearch_params();
155
    $self->store(
156
        Catmandu::Store::ElasticSearch->new(
157
            %$params,
158
            index_settings => $self->get_elasticsearch_settings(),
159
            index_mappings => $self->get_elasticsearch_mappings(),
160
        )
161
    );
162
}
163
164
165
=head2 $indexer->do_reindex();
166
167
Performs a reindex based on records passed in
168
169
=cut
170
171
sub do_reindex {
172
    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
173
174
    _log(1, $verbose, "Indexing ".$self->index."\n");
175
176
    if ( scalar @$biblionumbers ) {
177
        if ( $self->index eq 'biblios' ) {
178
            $next = sub {
179
                my $r = shift @$biblionumbers;
180
                return () unless defined $r;
181
                return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
182
            };
183
        }
184
        elsif ( $self->index eq 'authorities' ) {
185
            $next = sub {
186
                my $r = shift @$biblionumbers;
187
                return () unless defined $r;
188
                my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
189
                return ($r, $a->record);
190
            };
191
        }
192
    }
193
    else {
194
        my $records = $self->index eq 'biblios' ?
195
            Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ?
196
                Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index";
197
        $next = sub { $records->next(); };
198
    }
199
200
warn
201
    $self->drop_index() if ( $delete );
202
203
    my $count        = 0;
204
    my $commit_count = $commit;
205
    my ( @id_buffer, @commit_buffer );
206
    while ( my $record = $next->() ) {
207
        my $id     = $record->id;
208
        my $record = $record->record;
209
        _log( 1, $verbose, "$id\n" );
210
        $count++;
211
212
        push @id_buffer,     $id;
213
        push @commit_buffer, $record;
214
        if ( !( --$commit_count ) ) {
215
            _log( 2, $verbose, "Committing...\n" );
216
            $self->update_index( \@id_buffer, \@commit_buffer );
217
            $commit_count  = $commit;
218
            @id_buffer     = ();
219
            @commit_buffer = ();
220
        }
221
    }
222
    # There are probably uncommitted records (beacuse we didn't hit exact commit count)
223
    $self->update_index( \@id_buffer, \@commit_buffer );
224
    _log( 1, $verbose, "$count records indexed.\n" );
225
}
226
227
228
sub _log {
229
    my ($level, $verbose, $msg) = @_;
230
    print $msg if ($verbose >= $level);
166
}
231
}
167
232
168
sub _sanitise_records {
233
sub _sanitise_records {
(-)a/misc/search_tools/rebuild_elastic_search.pl (-77 / +7 lines)
Lines 122-195 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here
122
sanity_check();
122
sanity_check();
123
123
124
my $next;
124
my $next;
125
print "$verbose verbosity";
125
if ($index_biblios) {
126
if ($index_biblios) {
126
    _log(1, "Indexing biblios\n");
127
    my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
127
    if (@biblionumbers) {
128
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
128
        $next = sub {
129
    $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers);
129
            my $r = shift @biblionumbers;
130
            return () unless defined $r;
131
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132
        };
133
    } else {
134
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
135
        $next = sub {
136
            $records->next();
137
        }
138
    }
139
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
140
}
130
}
141
if ($index_authorities) {
131
if ($index_authorities) {
142
    _log(1, "Indexing authorities\n");
132
    my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
143
    if (@biblionumbers) {
133
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
144
        $next = sub {
134
    $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers);
145
            my $r = shift @biblionumbers;
146
            return () unless defined $r;
147
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
148
            return ($r, $a->record);
149
        };
150
    } else {
151
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
152
        $next = sub {
153
            $records->next();
154
        }
155
    }
156
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
157
}
158
159
sub do_reindex {
160
    my ( $next, $index_name ) = @_;
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
164
165
        # We know it's safe to not recreate the indexer because update_index
166
        # hasn't been called yet.
167
        $indexer->drop_index();
168
    }
169
170
    my $count        = 0;
171
    my $commit_count = $commit;
172
    my ( @id_buffer, @commit_buffer );
173
    while ( my $record = $next->() ) {
174
        my $id     = $record->id;
175
        my $record = $record->record;
176
        _log( 1, "$id\n" );
177
        $count++;
178
179
        push @id_buffer,     $id;
180
        push @commit_buffer, $record;
181
        if ( !( --$commit_count ) ) {
182
            _log( 2, "Committing...\n" );
183
            $indexer->update_index( \@id_buffer, \@commit_buffer );
184
            $commit_count  = $commit;
185
            @id_buffer     = ();
186
            @commit_buffer = ();
187
        }
188
    }
189
190
    # There are probably uncommitted records
191
    $indexer->update_index( \@id_buffer, \@commit_buffer );
192
    _log( 1, "$count records indexed.\n" );
193
}
135
}
194
136
195
# Checks some basic stuff to ensure that it's sane before we start.
137
# Checks some basic stuff to ensure that it's sane before we start.
Lines 199-212 sub sanity_check { Link Here
199
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
141
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
200
}
142
}
201
143
202
# Output progress information.
203
#
204
#   _log($level, $msg);
205
#
206
# Will output $msg if the verbosity setting is set to $level or more. Will
207
# not include a trailing newline.
208
sub _log {
209
    my ($level, $msg) = @_;
210
211
    print $msg if ($verbose >= $level);
212
}
213
- 

Return to bug 18948