View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-2 / +28 lines)
Lines 101-120 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
108
108
109
The iterator is a Koha::MetadataIterator object.
109
The iterator is a Koha::MetadataIterator object.
110
110
111
Possible options are:
112
113
=over 4
114
115
=item C<slice>
116
117
slice may be defined as a hash of two values: index and count. index
118
is the slice number to process and count is total number of slices.
119
With this information the iterator returns just the given slice of
120
records instead of all.
121
122
=back
123
111
=cut
124
=cut
112
125
113
sub get_all_biblios_iterator {
126
sub get_all_biblios_iterator {
127
    my ($self, %options) = @_;
128
129
    my $search_terms = {};
130
    my ($slice_modulo, $slice_count);
131
    if ($options{slice}) {
132
        $slice_count = $options{slice}->{count};
133
        $slice_modulo = $options{slice}->{index};
134
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
135
136
        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
137
    }
138
114
    my $database = Koha::Database->new();
139
    my $database = Koha::Database->new();
115
    my $schema   = $database->schema();
140
    my $schema   = $database->schema();
116
    my $rs =
141
    my $rs =
117
      $schema->resultset('Biblio')->search( {},
142
      $schema->resultset('Biblio')->search(
143
        $search_terms,
118
        { columns => [qw/ biblionumber /] } );
144
        { columns => [qw/ biblionumber /] } );
119
    my $next_func = sub {
145
    my $next_func = sub {
120
        # Warn and skip bad records, otherwise we break the loop
146
        # Warn and skip bad records, otherwise we break the loop
(-)a/Koha/MetadataRecord/Authority.pm (-2 / +36 lines)
Lines 151-170 sub authorized_heading { Link Here
151
151
152
=head2 get_all_authorities_iterator
152
=head2 get_all_authorities_iterator
153
153
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
155
155
156
This will provide an iterator object that will, one by one, provide the
156
This will provide an iterator object that will, one by one, provide the
157
Koha::MetadataRecord::Authority of each authority.
157
Koha::MetadataRecord::Authority of each authority.
158
158
159
The iterator is a Koha::MetadataIterator object.
159
The iterator is a Koha::MetadataIterator object.
160
160
161
Possible options are:
162
163
=over 4
164
165
=item C<slice>
166
167
slice may be defined as a hash of two values: index and count. index
168
is the slice number to process and count is total number of slices.
169
With this information the iterator returns just the given slice of
170
records instead of all.
171
172
=back
173
161
=cut
174
=cut
162
175
163
sub get_all_authorities_iterator {
176
sub get_all_authorities_iterator {
177
    my ($self, %options) = @_;
178
179
    my $search_terms = {
180
        marcxml => { '!=', undef }
181
    };
182
    my ($slice_modulo, $slice_count);
183
    if ($options{slice}) {
184
        $slice_count = $options{slice}->{count};
185
        $slice_modulo = $options{slice}->{index};
186
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
187
188
        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
189
        $search_terms = {
190
            '-and' => [
191
                $search_terms,
192
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
193
            ]
194
        };
195
    }
196
164
    my $database = Koha::Database->new();
197
    my $database = Koha::Database->new();
165
    my $schema   = $database->schema();
198
    my $schema   = $database->schema();
166
    my $rs =
199
    my $rs =
167
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
200
      $schema->resultset('AuthHeader')->search(
201
        $search_terms,
168
        { columns => [qw/ authid authtypecode marcxml /] } );
202
        { columns => [qw/ authid authtypecode marcxml /] } );
169
    my $next_func = sub {
203
    my $next_func = sub {
170
        my $row = $rs->next();
204
        my $row = $rs->next();
(-)a/misc/search_tools/rebuild_elastic_search.pl (-33 / +111 lines)
Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-p|--processes>
68
69
Number of processes to use for indexing. This can be used to do more indexing
70
work in parallel on multicore systems. By default, a single process is used.
71
67
=item B<-v|--verbose>
72
=item B<-v|--verbose>
68
73
69
By default, this program only emits warnings and errors. This makes it talk
74
By default, this program only emits warnings and errors. This makes it talk
Lines 79-84 Full documentation. Link Here
79
84
80
=back
85
=back
81
86
87
=head1 IMPLEMENTATION
88
82
=cut
89
=cut
83
90
84
use autodie;
91
use autodie;
Lines 95-115 use Pod::Usage; Link Here
95
102
96
my $verbose = 0;
103
my $verbose = 0;
97
my $commit = 5000;
104
my $commit = 5000;
98
my ($delete, $help, $man);
105
my ($delete, $help, $man, $processes);
99
my ($index_biblios, $index_authorities);
106
my ($index_biblios, $index_authorities);
100
my (@biblionumbers);
107
my (@record_numbers);
101
108
102
$|=1; # flushes output
109
$|=1; # flushes output
103
110
104
GetOptions(
111
GetOptions(
105
    'c|commit=i'       => \$commit,
112
    'c|commit=i'    => \$commit,
106
    'd|delete'         => \$delete,
113
    'd|delete'      => \$delete,
107
    'a|authorities' => \$index_authorities,
114
    'a|authorities' => \$index_authorities,
108
    'b|biblios' => \$index_biblios,
115
    'b|biblios'     => \$index_biblios,
109
    'bn|bnumber=i' => \@biblionumbers,
116
    'bn|bnumber=i'  => \@record_numbers,
110
    'v|verbose+'       => \$verbose,
117
    'p|processes=i' => \$processes,
111
    'h|help'           => \$help,
118
    'v|verbose+'    => \$verbose,
112
    'man'              => \$man,
119
    'h|help'        => \$help,
120
    'man'           => \$man,
113
);
121
);
114
122
115
# Default is to do both
123
# Default is to do both
Lines 120-168 unless ($index_authorities || $index_biblios) { Link Here
120
pod2usage(1) if $help;
128
pod2usage(1) if $help;
121
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
129
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
122
130
123
sanity_check();
131
_sanity_check();
132
133
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
134
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
135
136
my $slice_index = 0;
137
my $slice_count = $processes // 1;
138
139
if ($slice_count > 1) {
140
    # Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
141
    $slice_index = 1;
142
    for (my $proc = 2; $proc <= $processes; $proc++) {
143
        my $pid = fork();
144
        die "Failed to fork a child process\n" unless defined $pid;
145
        if ($pid == 0) {
146
            # Child process, give it a slice to process
147
            $slice_index = $proc;
148
            last;
149
        }
150
    }
151
    # Fudge the commit count a bit to spread out the Elasticsearch commits
152
    $commit *= 1 + 0.10 * ($slice_index - 1);
153
}
154
155
my %iterator_options;
156
if ($slice_index) {
157
    _log(1, "Processing slice $slice_index of $slice_count\n");
158
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
159
}
124
160
125
my $next;
161
my $next;
126
if ($index_biblios) {
162
if ($index_biblios) {
127
    _log(1, "Indexing biblios\n");
163
    _log(1, "Indexing biblios\n");
128
    if (@biblionumbers) {
164
    if (@record_numbers) {
129
        $next = sub {
165
        $next = sub {
130
            my $r = shift @biblionumbers;
166
            my $r = shift @record_numbers;
131
            return () unless defined $r;
167
            return () unless defined $r;
132
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
168
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
133
        };
169
        };
134
    } else {
170
    } else {
135
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
171
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
136
        $next = sub {
172
        $next = sub {
137
            $records->next();
173
            $records->next();
138
        }
174
        }
139
    }
175
    }
140
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
176
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
141
}
177
}
142
if ($index_authorities) {
178
if ($index_authorities) {
143
    _log(1, "Indexing authorities\n");
179
    _log(1, "Indexing authorities\n");
144
    if (@biblionumbers) {
180
    if (@record_numbers) {
145
        $next = sub {
181
        $next = sub {
146
            my $r = shift @biblionumbers;
182
            my $r = shift @record_numbers;
147
            return () unless defined $r;
183
            return () unless defined $r;
148
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
184
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
149
            return ($r, $a->record);
185
            return ($r, $a->record);
150
        };
186
        };
151
    } else {
187
    } else {
152
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
188
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
153
        $next = sub {
189
        $next = sub {
154
            $records->next();
190
            $records->next();
155
        }
191
        }
156
    }
192
    }
157
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
193
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
158
}
194
}
159
195
160
sub do_reindex {
196
if ($slice_index == 1) {
161
    my ( $next, $index_name ) = @_;
197
    # Main process, wait for children
198
    for (my $proc = 2; $proc <= $processes; $proc++) {
199
        wait();
200
    }
201
}
202
203
=head2 _verify_index_state
204
205
    _                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
206
207
Checks the index state and recreates it if requested.
162
208
209
=cut
210
211
sub _verify_index_state {
212
    my ( $index_name, $recreate ) = @_;
213
214
    _log(1, "Checking state of $index_name index\n");
163
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
215
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
164
216
165
    if ($delete) {
217
    if ($recreate) {
218
        _log(1, "Dropping and recreating $index_name index\n");
166
        $indexer->drop_index() if $indexer->index_exists();
219
        $indexer->drop_index() if $indexer->index_exists();
167
        $indexer->create_index();
220
        $indexer->create_index();
168
    }
221
    }
Lines 175-180 sub do_reindex { Link Here
175
    } elsif ($indexer->is_index_status_recreate_required) {
228
    } elsif ($indexer->is_index_status_recreate_required) {
176
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
229
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
177
    }
230
    }
231
}
232
233
=head2 _do_reindex
234
235
    _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
236
237
Does the actual reindexing. $callback is a function that always returns the next record.
238
239
=cut
240
241
sub _do_reindex {
242
    my ( $next, $index_name ) = @_;
243
244
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
178
245
179
    my $count        = 0;
246
    my $count        = 0;
180
    my $commit_count = $commit;
247
    my $commit_count = $commit;
Lines 192-203 sub do_reindex { Link Here
192
        push @id_buffer,     $id;
259
        push @id_buffer,     $id;
193
        push @commit_buffer, $record;
260
        push @commit_buffer, $record;
194
        if ( !( --$commit_count ) ) {
261
        if ( !( --$commit_count ) ) {
195
            _log( 1, "Committing $commit records..." );
262
            _log( 1, "Committing $commit records...\n" );
196
            $indexer->update_index( \@id_buffer, \@commit_buffer );
263
            $indexer->update_index( \@id_buffer, \@commit_buffer );
197
            $commit_count  = $commit;
264
            $commit_count  = $commit;
198
            @id_buffer     = ();
265
            @id_buffer     = ();
199
            @commit_buffer = ();
266
            @commit_buffer = ();
200
            _log( 1, " done\n" );
267
            _log( 1, "Commit complete\n" );
201
        }
268
        }
202
    }
269
    }
203
270
Lines 207-227 sub do_reindex { Link Here
207
    _log( 1, "Total $count records indexed\n" );
274
    _log( 1, "Total $count records indexed\n" );
208
}
275
}
209
276
210
# Checks some basic stuff to ensure that it's sane before we start.
277
=head2 _sanity_check
211
sub sanity_check {
278
279
    _sanity_check();
280
281
Checks some basic stuff to ensure that it's sane before we start.
282
283
=cut
284
285
sub _sanity_check {
212
    # Do we have an elasticsearch block defined?
286
    # Do we have an elasticsearch block defined?
213
    my $conf = C4::Context->config('elasticsearch');
287
    my $conf = C4::Context->config('elasticsearch');
214
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
288
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
215
}
289
}
216
290
217
# Output progress information.
291
=head2 _log
218
#
292
219
#   _log($level, $msg);
293
    _log($level, "Message\n");
220
#
294
221
# Will output $msg if the verbosity setting is set to $level or more. Will
295
Output progress information.
222
# not include a trailing newline.
296
297
Will output the message if verbosity level is set to $level or more. Will not
298
include a trailing newline automatically.
299
300
=cut
301
223
sub _log {
302
sub _log {
224
    my ($level, $msg) = @_;
303
    my ($level, $msg) = @_;
225
304
226
    print $msg if ($verbose >= $level);
305
    print "[$$] $msg" if ($verbose >= $level);
227
}
306
}
228
- 

Return to bug 21872