View | Details | Raw Unified | Return to bug 20384
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-7 / +12 lines)
Lines 101-107 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(\%options, $offset, $length);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
Lines 124-136 records instead of all. Link Here
124
=cut
124
=cut
125
125
126
sub get_all_biblios_iterator {
126
sub get_all_biblios_iterator {
127
    my ($self, %options) = @_;
127
    my ($self, $options, $offset, $length) = @_;
128
128
129
    my $search_terms = {};
129
    my $search_terms = {};
130
    my ($slice_modulo, $slice_count);
130
    my ($slice_modulo, $slice_count);
131
    if ($options{slice}) {
131
    if ($options->{slice}) {
132
        $slice_count = $options{slice}->{count};
132
        $slice_count = $options->{slice}->{count};
133
        $slice_modulo = $options{slice}->{index};
133
        $slice_modulo = $options->{slice}->{index};
134
134
        $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
135
        $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
135
    }
136
    }
136
137
Lines 138-145 sub get_all_biblios_iterator { Link Here
138
    my $schema   = $database->schema();
139
    my $schema   = $database->schema();
139
    my $rs =
140
    my $rs =
140
      $schema->resultset('Biblio')->search(
141
      $schema->resultset('Biblio')->search(
141
        $search_terms,
142
        $search_terms, {
142
        { columns => [qw/ biblionumber /] } );
143
        columns => [qw/ biblionumber /] ,
144
        offset => $offset ,
145
        rows => $length
146
      });
147
143
    my $next_func = sub {
148
    my $next_func = sub {
144
        # Warn and skip bad records, otherwise we break the loop
149
        # Warn and skip bad records, otherwise we break the loop
145
        while (1) {
150
        while (1) {
(-)a/Koha/MetadataRecord/Authority.pm (-8 / +13 lines)
Lines 151-157 sub authorized_heading { Link Here
151
151
152
=head2 get_all_authorities_iterator
152
=head2 get_all_authorities_iterator
153
153
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%options, $offset, $length);
155
155
156
This will provide an iterator object that will, one by one, provide the
156
This will provide an iterator object that will, one by one, provide the
157
Koha::MetadataRecord::Authority of each authority.
157
Koha::MetadataRecord::Authority of each authority.
Lines 174-188 records instead of all. Link Here
174
=cut
174
=cut
175
175
176
sub get_all_authorities_iterator {
176
sub get_all_authorities_iterator {
177
    my ($self, %options) = @_;
177
    my ($self, $options, $offset, $length) = @_;
178
178
179
    my $search_terms = {
179
    my $search_terms = {
180
        marcxml => { '!=', undef }
180
        marcxml => { '!=', undef }
181
    };
181
    };
182
    my ($slice_modulo, $slice_count);
182
    my ($slice_modulo, $slice_count);
183
    if ($options{slice}) {
183
    if ($options->{slice}) {
184
        $slice_count = $options{slice}->{count};
184
        $slice_count = $options->{slice}->{count};
185
        $slice_modulo = $options{slice}->{index};
185
        $slice_modulo = $options->{slice}->{index};
186
        $search_terms = {
186
        $search_terms = {
187
            '-and' => [
187
            '-and' => [
188
                %{$search_terms},
188
                %{$search_terms},
Lines 193-202 sub get_all_authorities_iterator { Link Here
193
193
194
    my $database = Koha::Database->new();
194
    my $database = Koha::Database->new();
195
    my $schema   = $database->schema();
195
    my $schema   = $database->schema();
196
196
    my $rs =
197
    my $rs =
197
      $schema->resultset('AuthHeader')->search(
198
        $schema->resultset('AuthHeader')->search(
198
        $search_terms,
199
            $search_terms,
199
        { columns => [qw/ authid authtypecode marcxml /] } );
200
            { columns => [qw/ authid authtypecode marcxml /],
201
            offset => $offset,
202
            rows => $length }
203
        );
204
200
    my $next_func = sub {
205
    my $next_func = sub {
201
        my $row = $rs->next();
206
        my $row = $rs->next();
202
        return if !$row;
207
        return if !$row;
(-)a/misc/search_tools/rebuild_elasticsearch.pl (-36 / +126 lines)
Lines 35-40 B<rebuild_elasticsearch.pl> Link Here
35
[B<-ai|--authid>]
35
[B<-ai|--authid>]
36
[B<-p|--processes>]
36
[B<-p|--processes>]
37
[B<-v|--verbose>]
37
[B<-v|--verbose>]
38
[B<-d|--delete>]
39
[B<-a|--authorities>]
40
[B<-b|--biblios>]
41
[B<-f|--file>]
42
[B<-bn|--bnnumber>]
43
[B<--length>]
44
[B<--offset>]
38
[B<-h|--help>]
45
[B<-h|--help>]
39
[B<--man>]
46
[B<--man>]
40
47
Lines 70-90 specifying neither and so both get indexed. Link Here
70
Index the biblios only. Combining this with B<-a> is the same as
77
Index the biblios only. Combining this with B<-a> is the same as
71
specifying neither and so both get indexed.
78
specifying neither and so both get indexed.
72
79
80
=item B<-f|--file>
81
82
Read a csv file created with a sql query, the csv must contain a column of biblionumber from biblios
83
or a column of authid from auth_header, then index biblios or authorities, but not both.
84
With this option you can't have options B<--length>, B<--offset>, and B<-bn>.
85
73
=item B<-bn|--bnumber>
86
=item B<-bn|--bnumber>
74
87
75
Only index the supplied biblionumber, mostly for testing purposes. May be
88
Only index the supplied biblionumber, mostly for testing purposes. May be
76
repeated.
89
repeated. Cannot be combined with options B<--offset> and B<--length>.
77
90
78
=item B<-ai|--authid>
91
=item B<-ai|--authid>
79
92
80
Only index the supplied authority id, mostly for testing purposes. May be
93
Only index the supplied authority id, mostly for testing purposes. May be
81
repeated.
94
repeated. Cannot be combined with options B<--offset> and B<--length>.
82
95
83
=item B<-p|--processes>
96
=item B<-p|--processes>
84
97
85
Number of processes to use for indexing. This can be used to do more indexing
98
Number of processes to use for indexing. This can be used to do more indexing
86
work in parallel on multicore systems. By default, a single process is used.
99
work in parallel on multicore systems. By default, a single process is used.
87
100
101
=item B<--offset>
102
103
Specify the row number for the first row to be indexed.
104
With this option you have to choose between biblios B<-b> or authorities B<-a>.
105
106
=item B<--length>
107
108
Specify the number of rows to be indexed.
109
With this option you have to choose between biblios B<-b> or authorities B<-a>.
110
88
=item B<-v|--verbose>
111
=item B<-v|--verbose>
89
112
90
By default, this program only emits warnings and errors. This makes it talk
113
By default, this program only emits warnings and errors. This makes it talk
Lines 116-127 use MARC::Field; Link Here
116
use MARC::Record;
139
use MARC::Record;
117
use Modern::Perl;
140
use Modern::Perl;
118
use Pod::Usage;
141
use Pod::Usage;
142
use Text::CSV;
119
143
120
my $verbose = 0;
144
my $verbose = 0;
121
my $commit = 5000;
145
my $commit = 5000;
122
my ($delete, $reset, $help, $man, $processes);
146
my ($delete, $reset, $help, $man, $processes);
123
my ($index_biblios, $index_authorities);
147
my ($index_biblios, $index_authorities);
124
my (@biblionumbers,@authids);
148
my (@biblionumbers,@authids);
149
my ($offset, $length, $file);
125
150
126
$|=1; # flushes output
151
$|=1; # flushes output
127
152
Lines 135-144 GetOptions( Link Here
135
    'ai|authid=i'  => \@authids,
160
    'ai|authid=i'  => \@authids,
136
    'p|processes=i' => \$processes,
161
    'p|processes=i' => \$processes,
137
    'v|verbose+'    => \$verbose,
162
    'v|verbose+'    => \$verbose,
163
    'offset=i'      => \$offset,
164
    'length=i'      => \$length,
165
    'f|file=s'      => \$file,
138
    'h|help'        => \$help,
166
    'h|help'        => \$help,
139
    'man'           => \$man,
167
    'man'           => \$man,
168
140
);
169
);
141
170
171
pod2usage(1) if $help;
172
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
173
142
# Default is to do both
174
# Default is to do both
143
unless ($index_authorities || $index_biblios) {
175
unless ($index_authorities || $index_biblios) {
144
    $index_authorities = $index_biblios = 1;
176
    $index_authorities = $index_biblios = 1;
Lines 148-165 if ($processes && ( @biblionumbers || @authids) ) { Link Here
148
    die "Argument p|processes cannot be combined with bn|bnumber or ai|authid";
180
    die "Argument p|processes cannot be combined with bn|bnumber or ai|authid";
149
}
181
}
150
182
151
pod2usage(1) if $help;
183
if ($file){
152
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
184
    if ($offset || $length || @biblionumbers || @authids){
185
        die "With options --file, you can't have options --length, --offset, -bn, and -ai" . "\n";
186
    }
187
}
153
188
154
_sanity_check();
189
if (@biblionumbers || @authids){
190
    if ($length || $offset){
191
        die "With option -bn or -ai, you can't have options --length, --offset" . "\n";
192
    }
193
}
155
194
156
if ($reset){
195
if ($reset){
157
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
196
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
158
    $delete = 1;
197
    $delete = 1;
159
}
198
}
160
199
161
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
200
if ($offset || $length){
162
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
201
    if ( $index_biblios && $index_authorities){
202
        die "With options --offset or --length, you have to choose between biblios (-b) or authorities (-a)." . "\n";
203
    }
204
}
205
206
_sanity_check();
163
207
164
my $slice_index = 0;
208
my $slice_index = 0;
165
my $slice_count = ( $processes //= 1 );
209
my $slice_count = ( $processes //= 1 );
Lines 184-221 if ($slice_count > 1) { Link Here
184
}
228
}
185
229
186
my $next;
230
my $next;
187
if ($index_biblios) {
231
if ($file){
188
    _log(1, "Indexing biblios\n");
232
    my @bibnums;
189
    if (@biblionumbers) {
233
    my $i = 0;
190
        $next = sub {
234
    my $input = $file;
191
            my $r = shift @biblionumbers;
235
    my $parser = Text::CSV->new();
192
            return () unless defined $r;
236
    open ( my $csv, "<", $input );
193
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
237
194
        };
238
    while (<$csv>) {
195
    } else {
239
        if ($parser->parse($_)){
196
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
240
            my @columns = $parser->fields();
197
        $next = sub {
241
            @bibnums[$i] = @columns;
198
            $records->next();
242
        $i++;
199
        }
243
        }
200
    }
244
    }
201
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
245
    close $csv;
202
}
246
203
if ($index_authorities) {
247
    if (@bibnums){
204
    _log(1, "Indexing authorities\n");
248
        my $type = shift @bibnums;
205
    if (@authids) {
249
        if ($type eq "biblionumber"){
206
        $next = sub {
250
            _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
207
            my $r = shift @authids;
251
            _log(1, "Indexing biblios\n");
208
            return () unless defined $r;
252
            $next = sub {
209
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
253
                my $bibnum = shift @bibnums;
210
            return ($r, $a);
254
                return () unless defined $bibnum;
211
        };
255
                return ($bibnum, Koha::BiblioUtils->get_from_biblionumber($bibnum, item_data => 1 ));
212
    } else {
256
            };
213
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
257
            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
214
        $next = sub {
258
215
            $records->next();
259
        } elsif ($type eq "authid"){
260
            _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
261
            _log(1, "Indexing authorities\n");
262
            $next = sub {
263
                my $authid = shift @bibnums;
264
                return () unless defined $authid;
265
                return ($authid,Koha::MetadataRecord::Authority->get_from_authid($authid));
266
            };
267
            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
268
        }
269
    }
270
271
} else {
272
    if ($index_biblios) {
273
        _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
274
        _log(1, "Indexing biblios\n");
275
276
        if (@biblionumbers) {
277
            $next = sub {
278
                my $r = shift @biblionumbers;
279
                return () unless defined $r;
280
                return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
281
            };
282
        }  else {
283
            my $records = Koha::BiblioUtils->get_all_biblios_iterator(\%iterator_options,$offset, $length);
284
            $next = sub {
285
                $records->next();
286
            }
287
        }
288
        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
289
    }
290
    if ($index_authorities) {
291
        _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
292
        _log(1, "Indexing authorities\n");
293
294
        if (@authids) {
295
            $next = sub {
296
                my $r = shift @authids;
297
                return () unless defined $r;
298
                my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
299
                return ($r, $a);
300
            };
301
302
        } else {
303
            my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%iterator_options,$offset, $length);
304
            $next = sub {
305
                $records->next();
306
            }
216
        }
307
        }
308
        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
217
    }
309
    }
218
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
219
}
310
}
220
311
221
if ($slice_index == 0) {
312
if ($slice_index == 0) {
222
- 

Return to bug 20384