View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-2 / +28 lines)
Lines 101-120 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
108
108
109
The iterator is a Koha::MetadataIterator object.
109
The iterator is a Koha::MetadataIterator object.
110
110
111
Possible options are:
112
113
=over 4
114
115
=item C<slice>
116
117
slice may be defined as a hash of two values: index and count. index
118
is the slice number to process and count is total number of slices.
119
With this information the iterator returns just the given slice of
120
records instead of all.
121
122
=back
123
111
=cut
124
=cut
112
125
113
sub get_all_biblios_iterator {
126
sub get_all_biblios_iterator {
127
    my ($self, %options) = @_;
128
129
    my $search_terms = {};
130
    my ($slice_modulo, $slice_count);
131
    if ($options{slice}) {
132
        $slice_count = $options{slice}->{count};
133
        $slice_modulo = $options{slice}->{index};
134
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
135
136
        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
137
    }
138
114
    my $database = Koha::Database->new();
139
    my $database = Koha::Database->new();
115
    my $schema   = $database->schema();
140
    my $schema   = $database->schema();
116
    my $rs =
141
    my $rs =
117
      $schema->resultset('Biblio')->search( {},
142
      $schema->resultset('Biblio')->search(
143
        $search_terms,
118
        { columns => [qw/ biblionumber /] } );
144
        { columns => [qw/ biblionumber /] } );
119
    my $next_func = sub {
145
    my $next_func = sub {
120
        # Warn and skip bad records, otherwise we break the loop
146
        # Warn and skip bad records, otherwise we break the loop
(-)a/Koha/MetadataRecord/Authority.pm (-2 / +36 lines)
Lines 150-169 sub authorized_heading { Link Here
150
150
151
=head2 get_all_authorities_iterator
151
=head2 get_all_authorities_iterator
152
152
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
154
155
This will provide an iterator object that will, one by one, provide the
155
This will provide an iterator object that will, one by one, provide the
156
Koha::MetadataRecord::Authority of each authority.
156
Koha::MetadataRecord::Authority of each authority.
157
157
158
The iterator is a Koha::MetadataIterator object.
158
The iterator is a Koha::MetadataIterator object.
159
159
160
Possible options are:
161
162
=over 4
163
164
=item C<slice>
165
166
slice may be defined as a hash of two values: index and count. index
167
is the slice number to process and count is total number of slices.
168
With this information the iterator returns just the given slice of
169
records instead of all.
170
171
=back
172
160
=cut
173
=cut
161
174
162
sub get_all_authorities_iterator {
175
sub get_all_authorities_iterator {
176
    my ($self, %options) = @_;
177
178
    my $search_terms = {
179
        marcxml => { '!=', undef }
180
    };
181
    my ($slice_modulo, $slice_count);
182
    if ($options{slice}) {
183
        $slice_count = $options{slice}->{count};
184
        $slice_modulo = $options{slice}->{index};
185
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
186
187
        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
188
        $search_terms = {
189
            '-and' => [
190
                $search_terms,
191
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
192
            ]
193
        };
194
    }
195
163
    my $database = Koha::Database->new();
196
    my $database = Koha::Database->new();
164
    my $schema   = $database->schema();
197
    my $schema   = $database->schema();
165
    my $rs =
198
    my $rs =
166
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
199
      $schema->resultset('AuthHeader')->search(
200
        $search_terms,
167
        { columns => [qw/ authid authtypecode marcxml /] } );
201
        { columns => [qw/ authid authtypecode marcxml /] } );
168
    my $next_func = sub {
202
    my $next_func = sub {
169
        my $row = $rs->next();
203
        my $row = $rs->next();
(-)a/misc/search_tools/rebuild_elastic_search.pl (-33 / +111 lines)
Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-p|--processes>
68
69
Number of processes to use for indexing. This can be used to do more indexing
70
work in parallel on multicore systems. By default, a single process is used.
71
67
=item B<-v|--verbose>
72
=item B<-v|--verbose>
68
73
69
By default, this program only emits warnings and errors. This makes it talk
74
By default, this program only emits warnings and errors. This makes it talk
Lines 79-84 Full documentation. Link Here
79
84
80
=back
85
=back
81
86
87
=head1 IMPLEMENTATION
88
82
=cut
89
=cut
83
90
84
use autodie;
91
use autodie;
Lines 94-114 use Pod::Usage; Link Here
94
101
95
my $verbose = 0;
102
my $verbose = 0;
96
my $commit = 5000;
103
my $commit = 5000;
97
my ($delete, $help, $man);
104
my ($delete, $help, $man, $processes);
98
my ($index_biblios, $index_authorities);
105
my ($index_biblios, $index_authorities);
99
my (@biblionumbers);
106
my (@record_numbers);
100
107
101
$|=1; # flushes output
108
$|=1; # flushes output
102
109
103
GetOptions(
110
GetOptions(
104
    'c|commit=i'       => \$commit,
111
    'c|commit=i'    => \$commit,
105
    'd|delete'         => \$delete,
112
    'd|delete'      => \$delete,
106
    'a|authorities' => \$index_authorities,
113
    'a|authorities' => \$index_authorities,
107
    'b|biblios' => \$index_biblios,
114
    'b|biblios'     => \$index_biblios,
108
    'bn|bnumber=i' => \@biblionumbers,
115
    'bn|bnumber=i'  => \@record_numbers,
109
    'v|verbose+'       => \$verbose,
116
    'p|processes=i' => \$processes,
110
    'h|help'           => \$help,
117
    'v|verbose+'    => \$verbose,
111
    'man'              => \$man,
118
    'h|help'        => \$help,
119
    'man'           => \$man,
112
);
120
);
113
121
114
# Default is to do both
122
# Default is to do both
Lines 119-167 unless ($index_authorities || $index_biblios) { Link Here
119
pod2usage(1) if $help;
127
pod2usage(1) if $help;
120
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
128
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
121
129
122
sanity_check();
130
_sanity_check();
131
132
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
133
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
134
135
my $slice_index = 0;
136
my $slice_count = $processes // 1;
137
138
if ($slice_count > 1) {
139
    # Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
140
    $slice_index = 1;
141
    for (my $proc = 2; $proc <= $processes; $proc++) {
142
        my $pid = fork();
143
        die "Failed to fork a child process\n" unless defined $pid;
144
        if ($pid == 0) {
145
            # Child process, give it a slice to process
146
            $slice_index = $proc;
147
            last;
148
        }
149
    }
150
    # Fudge the commit count a bit to spread out the Elasticsearch commits
151
    $commit *= 1 + 0.10 * ($slice_index - 1);
152
}
153
154
my %iterator_options;
155
if ($slice_index) {
156
    _log(1, "Processing slice $slice_index of $slice_count\n");
157
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
158
}
123
159
124
my $next;
160
my $next;
125
if ($index_biblios) {
161
if ($index_biblios) {
126
    _log(1, "Indexing biblios\n");
162
    _log(1, "Indexing biblios\n");
127
    if (@biblionumbers) {
163
    if (@record_numbers) {
128
        $next = sub {
164
        $next = sub {
129
            my $r = shift @biblionumbers;
165
            my $r = shift @record_numbers;
130
            return () unless defined $r;
166
            return () unless defined $r;
131
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
167
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132
        };
168
        };
133
    } else {
169
    } else {
134
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
170
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135
        $next = sub {
171
        $next = sub {
136
            $records->next();
172
            $records->next();
137
        }
173
        }
138
    }
174
    }
139
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
175
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
140
}
176
}
141
if ($index_authorities) {
177
if ($index_authorities) {
142
    _log(1, "Indexing authorities\n");
178
    _log(1, "Indexing authorities\n");
143
    if (@biblionumbers) {
179
    if (@record_numbers) {
144
        $next = sub {
180
        $next = sub {
145
            my $r = shift @biblionumbers;
181
            my $r = shift @record_numbers;
146
            return () unless defined $r;
182
            return () unless defined $r;
147
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
183
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
148
            return ($r, $a->record);
184
            return ($r, $a->record);
149
        };
185
        };
150
    } else {
186
    } else {
151
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
187
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152
        $next = sub {
188
        $next = sub {
153
            $records->next();
189
            $records->next();
154
        }
190
        }
155
    }
191
    }
156
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
192
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
157
}
193
}
158
194
159
sub do_reindex {
195
if ($slice_index == 1) {
160
    my ( $next, $index_name ) = @_;
196
    # Main process, wait for children
197
    for (my $proc = 2; $proc <= $processes; $proc++) {
198
        wait();
199
    }
200
}
201
202
=head2 _verify_index_state
203
204
    _                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
205
206
Checks the index state and recreates it if requested.
161
207
208
=cut
209
210
sub _verify_index_state {
211
    my ( $index_name, $recreate ) = @_;
212
213
    _log(1, "Checking state of $index_name index\n");
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
214
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
215
164
    if ($delete) {
216
    if ($recreate) {
217
        _log(1, "Dropping and recreating $index_name index\n");
165
        $indexer->drop_index() if $indexer->index_exists();
218
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
219
        $indexer->create_index();
167
    }
220
    }
Lines 174-179 sub do_reindex { Link Here
174
    } elsif ($indexer->is_index_status_recreate_required) {
227
    } elsif ($indexer->is_index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
228
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176
    }
229
    }
230
}
231
232
=head2 _do_reindex
233
234
    _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
235
236
Does the actual reindexing. $callback is a function that always returns the next record.
237
238
=cut
239
240
sub _do_reindex {
241
    my ( $next, $index_name ) = @_;
242
243
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
177
244
178
    my $count        = 0;
245
    my $count        = 0;
179
    my $commit_count = $commit;
246
    my $commit_count = $commit;
Lines 191-202 sub do_reindex { Link Here
191
        push @id_buffer,     $id;
258
        push @id_buffer,     $id;
192
        push @commit_buffer, $record;
259
        push @commit_buffer, $record;
193
        if ( !( --$commit_count ) ) {
260
        if ( !( --$commit_count ) ) {
194
            _log( 1, "Committing $commit records..." );
261
            _log( 1, "Committing $commit records...\n" );
195
            $indexer->update_index( \@id_buffer, \@commit_buffer );
262
            $indexer->update_index( \@id_buffer, \@commit_buffer );
196
            $commit_count  = $commit;
263
            $commit_count  = $commit;
197
            @id_buffer     = ();
264
            @id_buffer     = ();
198
            @commit_buffer = ();
265
            @commit_buffer = ();
199
            _log( 1, " done\n" );
266
            _log( 1, "Commit complete\n" );
200
        }
267
        }
201
    }
268
    }
202
269
Lines 206-226 sub do_reindex { Link Here
206
    _log( 1, "Total $count records indexed\n" );
273
    _log( 1, "Total $count records indexed\n" );
207
}
274
}
208
275
209
# Checks some basic stuff to ensure that it's sane before we start.
276
=head2 _sanity_check
210
sub sanity_check {
277
278
    _sanity_check();
279
280
Checks some basic stuff to ensure that it's sane before we start.
281
282
=cut
283
284
sub _sanity_check {
211
    # Do we have an elasticsearch block defined?
285
    # Do we have an elasticsearch block defined?
212
    my $conf = C4::Context->config('elasticsearch');
286
    my $conf = C4::Context->config('elasticsearch');
213
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
287
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
214
}
288
}
215
289
216
# Output progress information.
290
=head2 _log
217
#
291
218
#   _log($level, $msg);
292
    _log($level, "Message\n");
219
#
293
220
# Will output $msg if the verbosity setting is set to $level or more. Will
294
Output progress information.
221
# not include a trailing newline.
295
296
Will output the message if verbosity level is set to $level or more. Will not
297
include a trailing newline automatically.
298
299
=cut
300
222
sub _log {
301
sub _log {
223
    my ($level, $msg) = @_;
302
    my ($level, $msg) = @_;
224
303
225
    print $msg if ($verbose >= $level);
304
    print "[$$] $msg" if ($verbose >= $level);
226
}
305
}
227
- 

Return to bug 21872