View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-2 / +26 lines)
Lines 101-120 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
108
108
109
The iterator is a Koha::MetadataIterator object.
109
The iterator is a Koha::MetadataIterator object.
110
110
111
Possible options are:
112
113
=over 4
114
115
=item C<slice>
116
117
slice may be defined as a hash of two values: index and count. index
118
is the slice number to process and count is total number of slices.
119
With this information the iterator returns just the given slice of
120
records instead of all.
121
111
=cut
122
=cut
112
123
113
sub get_all_biblios_iterator {
124
sub get_all_biblios_iterator {
125
    my ($self, %options) = @_;
126
127
    my $search_terms = {};
128
    my ($slice_modulo, $slice_count);
129
    if ($options{slice}) {
130
        $slice_count = $options{slice}->{count};
131
        $slice_modulo = $options{slice}->{index};
132
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
133
134
        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
135
    }
136
114
    my $database = Koha::Database->new();
137
    my $database = Koha::Database->new();
115
    my $schema   = $database->schema();
138
    my $schema   = $database->schema();
116
    my $rs =
139
    my $rs =
117
      $schema->resultset('Biblio')->search( {},
140
      $schema->resultset('Biblio')->search(
141
        $search_terms,
118
        { columns => [qw/ biblionumber /] } );
142
        { columns => [qw/ biblionumber /] } );
119
    my $next_func = sub {
143
    my $next_func = sub {
120
        # Warn and skip bad records, otherwise we break the loop
144
        # Warn and skip bad records, otherwise we break the loop
(-)a/Koha/MetadataRecord/Authority.pm (-2 / +34 lines)
Lines 150-169 sub authorized_heading { Link Here
150
150
151
=head2 get_all_authorities_iterator
151
=head2 get_all_authorities_iterator
152
152
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
154
155
This will provide an iterator object that will, one by one, provide the
155
This will provide an iterator object that will, one by one, provide the
156
Koha::MetadataRecord::Authority of each authority.
156
Koha::MetadataRecord::Authority of each authority.
157
157
158
The iterator is a Koha::MetadataIterator object.
158
The iterator is a Koha::MetadataIterator object.
159
159
160
Possible options are:
161
162
=over 4
163
164
=item C<slice>
165
166
slice may be defined as a hash of two values: index and count. index
167
is the slice number to process and count is total number of slices.
168
With this information the iterator returns just the given slice of
169
records instead of all.
170
160
=cut
171
=cut
161
172
162
sub get_all_authorities_iterator {
173
sub get_all_authorities_iterator {
174
    my ($self, %options) = @_;
175
176
    my $search_terms = {
177
        marcxml => { '!=', undef }
178
    };
179
    my ($slice_modulo, $slice_count);
180
    if ($options{slice}) {
181
        $slice_count = $options{slice}->{count};
182
        $slice_modulo = $options{slice}->{index};
183
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
184
185
        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
186
        $search_terms = {
187
            '-and' => [
188
                $search_terms,
189
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
190
            ]
191
        };
192
    }
193
163
    my $database = Koha::Database->new();
194
    my $database = Koha::Database->new();
164
    my $schema   = $database->schema();
195
    my $schema   = $database->schema();
165
    my $rs =
196
    my $rs =
166
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
197
      $schema->resultset('AuthHeader')->search(
198
        $search_terms,
167
        { columns => [qw/ authid authtypecode marcxml /] } );
199
        { columns => [qw/ authid authtypecode marcxml /] } );
168
    my $next_func = sub {
200
    my $next_func = sub {
169
        my $row = $rs->next();
201
        my $row = $rs->next();
(-)a/misc/search_tools/rebuild_elastic_search.pl (-33 / +109 lines)
Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-p|--processes>
68
69
Number of processes to use for indexing. This can be used to do more indexing
70
work in parallel on multicore systems. By default, a single process is used.
71
67
=item B<-v|--verbose>
72
=item B<-v|--verbose>
68
73
69
By default, this program only emits warnings and errors. This makes it talk
74
By default, this program only emits warnings and errors. This makes it talk
Lines 94-114 use Pod::Usage; Link Here
94
99
95
my $verbose = 0;
100
my $verbose = 0;
96
my $commit = 5000;
101
my $commit = 5000;
97
my ($delete, $help, $man);
102
my ($delete, $help, $man, $processes);
98
my ($index_biblios, $index_authorities);
103
my ($index_biblios, $index_authorities);
99
my (@biblionumbers);
104
my (@record_numbers);
100
105
101
$|=1; # flushes output
106
$|=1; # flushes output
102
107
103
GetOptions(
108
GetOptions(
104
    'c|commit=i'       => \$commit,
109
    'c|commit=i'    => \$commit,
105
    'd|delete'         => \$delete,
110
    'd|delete'      => \$delete,
106
    'a|authorities' => \$index_authorities,
111
    'a|authorities' => \$index_authorities,
107
    'b|biblios' => \$index_biblios,
112
    'b|biblios'     => \$index_biblios,
108
    'bn|bnumber=i' => \@biblionumbers,
113
    'bn|bnumber=i'  => \@record_numbers,
109
    'v|verbose+'       => \$verbose,
114
    'p|processes=i' => \$processes,
110
    'h|help'           => \$help,
115
    'v|verbose+'    => \$verbose,
111
    'man'              => \$man,
116
    'h|help'        => \$help,
117
    'man'           => \$man,
112
);
118
);
113
119
114
# Default is to do both
120
# Default is to do both
Lines 119-167 unless ($index_authorities || $index_biblios) { Link Here
119
pod2usage(1) if $help;
125
pod2usage(1) if $help;
120
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
126
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
121
127
122
sanity_check();
128
_sanity_check();
129
130
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
131
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
132
133
my $slice_index = 0;
134
my $slice_count = $processes // 1;
135
136
if ($slice_count > 1) {
137
    # Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
138
    $slice_index = 1;
139
    for (my $proc = 2; $proc <= $processes; $proc++) {
140
        my $pid = fork();
141
        die "Failed to fork a child process\n" unless defined $pid;
142
        if ($pid == 0) {
143
            # Child process, give it a slice to process
144
            $slice_index = $proc;
145
            last;
146
        }
147
    }
148
    # Fudge the commit count a bit to spread out the Elasticsearch commits
149
    $commit *= 1 + 0.10 * ($slice_index - 1);
150
}
151
152
my %iterator_options;
153
if ($slice_index) {
154
    _log(1, "Processing slice $slice_index of $slice_count\n");
155
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
156
}
123
157
124
my $next;
158
my $next;
125
if ($index_biblios) {
159
if ($index_biblios) {
126
    _log(1, "Indexing biblios\n");
160
    _log(1, "Indexing biblios\n");
127
    if (@biblionumbers) {
161
    if (@record_numbers) {
128
        $next = sub {
162
        $next = sub {
129
            my $r = shift @biblionumbers;
163
            my $r = shift @record_numbers;
130
            return () unless defined $r;
164
            return () unless defined $r;
131
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
165
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132
        };
166
        };
133
    } else {
167
    } else {
134
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
168
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135
        $next = sub {
169
        $next = sub {
136
            $records->next();
170
            $records->next();
137
        }
171
        }
138
    }
172
    }
139
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
173
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
140
}
174
}
141
if ($index_authorities) {
175
if ($index_authorities) {
142
    _log(1, "Indexing authorities\n");
176
    _log(1, "Indexing authorities\n");
143
    if (@biblionumbers) {
177
    if (@record_numbers) {
144
        $next = sub {
178
        $next = sub {
145
            my $r = shift @biblionumbers;
179
            my $r = shift @record_numbers;
146
            return () unless defined $r;
180
            return () unless defined $r;
147
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
181
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
148
            return ($r, $a->record);
182
            return ($r, $a->record);
149
        };
183
        };
150
    } else {
184
    } else {
151
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
185
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152
        $next = sub {
186
        $next = sub {
153
            $records->next();
187
            $records->next();
154
        }
188
        }
155
    }
189
    }
156
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
190
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
157
}
191
}
158
192
159
sub do_reindex {
193
if ($slice_index == 1) {
160
    my ( $next, $index_name ) = @_;
194
    # Main process, wait for children
195
    for (my $proc = 2; $proc <= $processes; $proc++) {
196
        wait();
197
    }
198
}
199
200
=head2 _verify_index_state
201
202
    _                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
203
204
Checks the index state and recreates it if requested.
161
205
206
=cut
207
208
sub _verify_index_state {
209
    my ( $index_name, $recreate ) = @_;
210
211
    _log(1, "Checking state of $index_name index\n");
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
212
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
213
164
    if ($delete) {
214
    if ($recreate) {
215
        _log(1, "Dropping and recreating $index_name index\n");
165
        $indexer->drop_index() if $indexer->index_exists();
216
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
217
        $indexer->create_index();
167
    }
218
    }
Lines 174-179 sub do_reindex { Link Here
174
    } elsif ($indexer->is_index_status_recreate_required) {
225
    } elsif ($indexer->is_index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
226
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176
    }
227
    }
228
}
229
230
=head2 _do_reindex
231
232
    _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
233
234
Does the actual reindexing. $callback is a function that always returns the next record.
235
236
=cut
237
238
sub _do_reindex {
239
    my ( $next, $index_name ) = @_;
240
241
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
177
242
178
    my $count        = 0;
243
    my $count        = 0;
179
    my $commit_count = $commit;
244
    my $commit_count = $commit;
Lines 191-202 sub do_reindex { Link Here
191
        push @id_buffer,     $id;
256
        push @id_buffer,     $id;
192
        push @commit_buffer, $record;
257
        push @commit_buffer, $record;
193
        if ( !( --$commit_count ) ) {
258
        if ( !( --$commit_count ) ) {
194
            _log( 1, "Committing $commit records..." );
259
            _log( 1, "Committing $commit records...\n" );
195
            $indexer->update_index( \@id_buffer, \@commit_buffer );
260
            $indexer->update_index( \@id_buffer, \@commit_buffer );
196
            $commit_count  = $commit;
261
            $commit_count  = $commit;
197
            @id_buffer     = ();
262
            @id_buffer     = ();
198
            @commit_buffer = ();
263
            @commit_buffer = ();
199
            _log( 1, " done\n" );
264
            _log( 1, "Commit complete\n" );
200
        }
265
        }
201
    }
266
    }
202
267
Lines 206-226 sub do_reindex { Link Here
206
    _log( 1, "Total $count records indexed\n" );
271
    _log( 1, "Total $count records indexed\n" );
207
}
272
}
208
273
209
# Checks some basic stuff to ensure that it's sane before we start.
274
=head2 _sanity_check
210
sub sanity_check {
275
276
    _sanity_check();
277
278
Checks some basic stuff to ensure that it's sane before we start.
279
280
=cut
281
282
sub _sanity_check {
211
    # Do we have an elasticsearch block defined?
283
    # Do we have an elasticsearch block defined?
212
    my $conf = C4::Context->config('elasticsearch');
284
    my $conf = C4::Context->config('elasticsearch');
213
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
285
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
214
}
286
}
215
287
216
# Output progress information.
288
=head2 _log
217
#
289
218
#   _log($level, $msg);
290
    _log($level, "Message\n");
219
#
291
220
# Will output $msg if the verbosity setting is set to $level or more. Will
292
Output progress information.
221
# not include a trailing newline.
293
294
Will output the message if verbosity level is set to $level or more. Will not 
295
include a trailing newline automatically.
296
297
=cut
298
222
sub _log {
299
sub _log {
223
    my ($level, $msg) = @_;
300
    my ($level, $msg) = @_;
224
301
225
    print $msg if ($verbose >= $level);
302
    print "[$$] $msg" if ($verbose >= $level);
226
}
303
}
227
- 

Return to bug 21872