View | Details | Raw Unified | Return to bug 20384
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-7 / +12 lines)
Lines 101-107 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(\%options, $offset, $length);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
Lines 124-136 records instead of all. Link Here
124
=cut
124
=cut
125
125
126
sub get_all_biblios_iterator {
126
sub get_all_biblios_iterator {
127
    my ($self, %options) = @_;
127
    my ($self, $options, $offset, $length) = @_;
128
128
129
    my $search_terms = {};
129
    my $search_terms = {};
130
    my ($slice_modulo, $slice_count);
130
    my ($slice_modulo, $slice_count);
131
    if ($options{slice}) {
131
    if ($options->{slice}) {
132
        $slice_count = $options{slice}->{count};
132
        $slice_count = $options->{slice}->{count};
133
        $slice_modulo = $options{slice}->{index};
133
        $slice_modulo = $options->{slice}->{index};
134
134
        $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
135
        $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
135
    }
136
    }
136
137
Lines 138-145 sub get_all_biblios_iterator { Link Here
138
    my $schema   = $database->schema();
139
    my $schema   = $database->schema();
139
    my $rs =
140
    my $rs =
140
      $schema->resultset('Biblio')->search(
141
      $schema->resultset('Biblio')->search(
141
        $search_terms,
142
        $search_terms, {
142
        { columns => [qw/ biblionumber /] } );
143
        columns => [qw/ biblionumber /] ,
144
        offset => $offset ,
145
        rows => $length
146
      });
147
143
    my $next_func = sub {
148
    my $next_func = sub {
144
        # Warn and skip bad records, otherwise we break the loop
149
        # Warn and skip bad records, otherwise we break the loop
145
        while (1) {
150
        while (1) {
(-)a/Koha/MetadataRecord/Authority.pm (-8 / +13 lines)
Lines 151-157 sub authorized_heading { Link Here
151
151
152
=head2 get_all_authorities_iterator
152
=head2 get_all_authorities_iterator
153
153
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%options, $offset, $length);
155
155
156
This will provide an iterator object that will, one by one, provide the
156
This will provide an iterator object that will, one by one, provide the
157
Koha::MetadataRecord::Authority of each authority.
157
Koha::MetadataRecord::Authority of each authority.
Lines 174-188 records instead of all. Link Here
174
=cut
174
=cut
175
175
176
sub get_all_authorities_iterator {
176
sub get_all_authorities_iterator {
177
    my ($self, %options) = @_;
177
    my ($self, $options, $offset, $length) = @_;
178
178
179
    my $search_terms = {
179
    my $search_terms = {
180
        marcxml => { '!=', undef }
180
        marcxml => { '!=', undef }
181
    };
181
    };
182
    my ($slice_modulo, $slice_count);
182
    my ($slice_modulo, $slice_count);
183
    if ($options{slice}) {
183
    if ($options->{slice}) {
184
        $slice_count = $options{slice}->{count};
184
        $slice_count = $options->{slice}->{count};
185
        $slice_modulo = $options{slice}->{index};
185
        $slice_modulo = $options->{slice}->{index};
186
        $search_terms = {
186
        $search_terms = {
187
            '-and' => [
187
            '-and' => [
188
                %{$search_terms},
188
                %{$search_terms},
Lines 193-202 sub get_all_authorities_iterator { Link Here
193
193
194
    my $database = Koha::Database->new();
194
    my $database = Koha::Database->new();
195
    my $schema   = $database->schema();
195
    my $schema   = $database->schema();
196
196
    my $rs =
197
    my $rs =
197
      $schema->resultset('AuthHeader')->search(
198
        $schema->resultset('AuthHeader')->search(
198
        $search_terms,
199
            $search_terms,
199
        { columns => [qw/ authid authtypecode marcxml /] } );
200
            { columns => [qw/ authid authtypecode marcxml /],
201
            offset => $offset,
202
            rows => $length }
203
        );
204
200
    my $next_func = sub {
205
    my $next_func = sub {
201
        my $row = $rs->next();
206
        my $row = $rs->next();
202
        return if !$row;
207
        return if !$row;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +1 lines)
Lines 399-405 sub _sanitise_records { Link Here
399
        if ( $rec ) {
399
        if ( $rec ) {
400
            $rec->delete_fields($rec->field('999'));
400
            $rec->delete_fields($rec->field('999'));
401
            # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs.
401
            # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs.
402
            $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum));
402
            $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum)) if ($bibnum);
403
        }
403
        }
404
    }
404
    }
405
}
405
}
(-)a/misc/search_tools/rebuild_elasticsearch.pl (-36 / +127 lines)
Lines 28-33 rebuild_elasticsearch.pl - inserts records from a Koha database into Elasticsear Link Here
28
B<rebuild_elasticsearch.pl>
28
B<rebuild_elasticsearch.pl>
29
[B<-c|--commit>=C<count>]
29
[B<-c|--commit>=C<count>]
30
[B<-v|--verbose>]
30
[B<-v|--verbose>]
31
[B<-d|--delete>]
32
[B<-a|--authorities>]
33
[B<-b|--biblios>]
34
[B<-f|--file>]
35
[B<-bn|--bnnumber>]
36
[B<--length>]
37
[B<--offset>]
31
[B<-h|--help>]
38
[B<-h|--help>]
32
[B<--man>]
39
[B<--man>]
33
40
Lines 58-74 specifying neither and so both get indexed. Link Here
58
Index the biblios only. Combining this with B<-a> is the same as
65
Index the biblios only. Combining this with B<-a> is the same as
59
specifying neither and so both get indexed.
66
specifying neither and so both get indexed.
60
67
68
=item B<-f|--file>
69
70
Read a csv file created with a sql query, the csv must contain a column of biblionumber from biblios
71
or a column of authid from auth_header, then index biblios or authorities, but not both.
72
With this option you can't have options B<--length>, B<--offset>, and B<-bn>.
73
61
=item B<-bn|--bnumber>
74
=item B<-bn|--bnumber>
62
75
63
Only index the supplied biblionumber, mostly for testing purposes. May be
76
Only index the supplied biblionumber, mostly for testing purposes. May be
64
repeated. This also applies to authorities via authid, so if you're using it,
77
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
78
you probably only want to do one or the other at a time,
79
with this option you can't have options B<--offset> and B<--length>.
66
80
67
=item B<-p|--processes>
81
=item B<-p|--processes>
68
82
69
Number of processes to use for indexing. This can be used to do more indexing
83
Number of processes to use for indexing. This can be used to do more indexing
70
work in parallel on multicore systems. By default, a single process is used.
84
work in parallel on multicore systems. By default, a single process is used.
71
85
86
=item B<--offset>
87
88
Specify the row number for the first row to be indexed.
89
With this option you have to choose between biblios B<-b> or authorities B<-a>.
90
91
=item B<--length>
92
93
Specify the number of rows to be indexed.
94
With this option you have to choose between biblios B<-b> or authorities B<-a>.
95
72
=item B<-v|--verbose>
96
=item B<-v|--verbose>
73
97
74
By default, this program only emits warnings and errors. This makes it talk
98
By default, this program only emits warnings and errors. This makes it talk
Lines 99-110 use MARC::Field; Link Here
99
use MARC::Record;
123
use MARC::Record;
100
use Modern::Perl;
124
use Modern::Perl;
101
use Pod::Usage;
125
use Pod::Usage;
126
use Text::CSV;
102
127
103
my $verbose = 0;
128
my $verbose = 0;
104
my $commit = 5000;
129
my $commit = 5000;
105
my ($delete, $help, $man, $processes);
130
my ($delete, $help, $man, $processes);
106
my ($index_biblios, $index_authorities);
131
my ($index_biblios, $index_authorities);
107
my (@record_numbers);
132
my @record_numbers;
133
my ($offset, $length, $file);
108
134
109
$|=1; # flushes output
135
$|=1; # flushes output
110
136
Lines 116-125 GetOptions( Link Here
116
    'bn|bnumber=i'  => \@record_numbers,
142
    'bn|bnumber=i'  => \@record_numbers,
117
    'p|processes=i' => \$processes,
143
    'p|processes=i' => \$processes,
118
    'v|verbose+'    => \$verbose,
144
    'v|verbose+'    => \$verbose,
145
    'offset=i'      => \$offset,
146
    'length=i'      => \$length,
147
    'f|file=s'      => \$file,
119
    'h|help'        => \$help,
148
    'h|help'        => \$help,
120
    'man'           => \$man,
149
    'man'           => \$man,
150
121
);
151
);
122
152
153
pod2usage(1) if $help;
154
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
155
123
# Default is to do both
156
# Default is to do both
124
unless ($index_authorities || $index_biblios) {
157
unless ($index_authorities || $index_biblios) {
125
    $index_authorities = $index_biblios = 1;
158
    $index_authorities = $index_biblios = 1;
Lines 129-141 if ($processes && @record_numbers) { Link Here
129
    die "Argument p|processes cannot be combined with bn|bnumber";
162
    die "Argument p|processes cannot be combined with bn|bnumber";
130
}
163
}
131
164
132
pod2usage(1) if $help;
165
if ($file){
133
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
166
    if ($offset || $length || @record_numbers){
167
        die "With options --file, you can't have options --length, --offset, and -bn" . "\n";
168
    }
169
}
134
170
135
_sanity_check();
171
if (@record_numbers){
172
    if ($length || $offset){
173
        die "With option -bn, you can't have options --length, --offset" . "\n";
174
    }
175
}
136
176
137
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
177
if ($offset || $length){
138
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
178
    if ( $index_biblios && $index_authorities){
179
        die "With options --offset or --length, you have to choose between biblios (-b) or authorities (-a)." . "\n";
180
    }
181
}
182
183
_sanity_check();
139
184
140
my $slice_index = 0;
185
my $slice_index = 0;
141
my $slice_count = ( $processes //= 1 );
186
my $slice_count = ( $processes //= 1 );
Lines 160-197 if ($slice_count > 1) { Link Here
160
}
205
}
161
206
162
my $next;
207
my $next;
163
if ($index_biblios) {
208
if ($file){
164
    _log(1, "Indexing biblios\n");
209
    my @bibnums;
165
    if (@record_numbers) {
210
    my $i = 0;
166
        $next = sub {
211
    my $input = $file;
167
            my $r = shift @record_numbers;
212
    my $parser = Text::CSV->new();
168
            return () unless defined $r;
213
    open ( my $csv, "<", $input );
169
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
214
170
        };
215
    while (<$csv>) {
171
    } else {
216
        if ($parser->parse($_)){
172
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
217
            my @columns = $parser->fields();
173
        $next = sub {
218
            @bibnums[$i] = @columns;
174
            $records->next();
219
        $i++;
175
        }
220
        }
176
    }
221
    }
177
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
222
    close $csv;
178
}
223
179
if ($index_authorities) {
224
    if (@bibnums){
180
    _log(1, "Indexing authorities\n");
225
        my $type = shift @bibnums;
181
    if (@record_numbers) {
226
        if ($type eq "biblionumber"){
182
        $next = sub {
227
            _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
183
            my $r = shift @record_numbers;
228
            _log(1, "Indexing biblios\n");
184
            return () unless defined $r;
229
            $next = sub {
185
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
230
                my $bibnum = shift @bibnums;
186
            return ($r, $a->record);
231
                return () unless defined $bibnum;
187
        };
232
                return ($bibnum, Koha::BiblioUtils->get_from_biblionumber($bibnum, item_data => 1 ));
188
    } else {
233
            };
189
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
234
            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
190
        $next = sub {
235
191
            $records->next();
236
        } elsif ($type eq "authid"){
237
            _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
238
            _log(1, "Indexing authorities\n");
239
            $next = sub {
240
                my $authid = shift @bibnums;
241
                return () unless defined $authid;
242
                return ($authid,Koha::MetadataRecord::Authority->get_from_authid($authid));
243
            };
244
            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
245
        }
246
    }
247
248
} else {
249
    if ($index_biblios) {
250
        _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
251
        _log(1, "Indexing biblios\n");
252
253
        if (@record_numbers) {
254
            my @bibnums = @record_numbers;
255
            $next = sub {
256
                my $r = shift @bibnums;
257
                return () unless defined $r;
258
                return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
259
            };
260
        }  else {
261
            my $records = Koha::BiblioUtils->get_all_biblios_iterator(\%iterator_options,$offset, $length);
262
            $next = sub {
263
                $records->next();
264
            }
265
        }
266
        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
267
    }
268
    if ($index_authorities) {
269
        _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
270
        _log(1, "Indexing authorities\n");
271
272
        if (@record_numbers) {
273
            my @authnums = @record_numbers;
274
            $next = sub {
275
                my $r = shift @authnums;
276
                return () unless defined $r;
277
                return ($r,Koha::MetadataRecord::Authority->get_from_authid($r));
278
            };
279
        } else {
280
            my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%iterator_options,$offset, $length);
281
            $next = sub {
282
                $records->next();
283
            }
192
        }
284
        }
285
        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
193
    }
286
    }
194
    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
195
}
287
}
196
288
197
if ($slice_index == 0) {
289
if ($slice_index == 0) {
198
- 

Return to bug 20384