View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-2 / +26 lines)
Lines 101-120 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
108
108
109
The iterator is a Koha::MetadataIterator object.
109
The iterator is a Koha::MetadataIterator object.
110
110
111
Possible options are:
112
113
=over 4
114
115
=item C<slice>
116
117
slice may be defined as a hash of two values: index and count. index
118
is the slice number to process and count is total number of slices.
119
With this information the iterator returns just the given slice of
120
records instead of all.
121
111
=cut
122
=cut
112
123
113
sub get_all_biblios_iterator {
124
sub get_all_biblios_iterator {
125
    my ($self, %options) = @_;
126
127
    my $search_terms = {};
128
    my ($slice_modulo, $slice_count);
129
    if ($options{slice}) {
130
        $slice_count = $options{slice}->{count};
131
        $slice_modulo = $options{slice}->{index};
132
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
133
134
        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
135
    }
136
114
    my $database = Koha::Database->new();
137
    my $database = Koha::Database->new();
115
    my $schema   = $database->schema();
138
    my $schema   = $database->schema();
116
    my $rs =
139
    my $rs =
117
      $schema->resultset('Biblio')->search( {},
140
      $schema->resultset('Biblio')->search(
141
        $search_terms,
118
        { columns => [qw/ biblionumber /] } );
142
        { columns => [qw/ biblionumber /] } );
119
    my $next_func = sub {
143
    my $next_func = sub {
120
        # Warn and skip bad records, otherwise we break the loop
144
        # Warn and skip bad records, otherwise we break the loop
(-)a/Koha/MetadataRecord/Authority.pm (-2 / +34 lines)
Lines 150-169 sub authorized_heading { Link Here
150
150
151
=head2 get_all_authorities_iterator
151
=head2 get_all_authorities_iterator
152
152
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
154
155
This will provide an iterator object that will, one by one, provide the
155
This will provide an iterator object that will, one by one, provide the
156
Koha::MetadataRecord::Authority of each authority.
156
Koha::MetadataRecord::Authority of each authority.
157
157
158
The iterator is a Koha::MetadataIterator object.
158
The iterator is a Koha::MetadataIterator object.
159
159
160
Possible options are:
161
162
=over 4
163
164
=item C<slice>
165
166
slice may be defined as a hash of two values: index and count. index
167
is the slice number to process and count is total number of slices.
168
With this information the iterator returns just the given slice of
169
records instead of all.
170
160
=cut
171
=cut
161
172
162
sub get_all_authorities_iterator {
173
sub get_all_authorities_iterator {
174
    my ($self, %options) = @_;
175
176
    my $search_terms = {
177
        marcxml => { '!=', undef }
178
    };
179
    my ($slice_modulo, $slice_count);
180
    if ($options{slice}) {
181
        $slice_count = $options{slice}->{count};
182
        $slice_modulo = $options{slice}->{index};
183
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
184
185
        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
186
        $search_terms = {
187
            '-and' => [
188
                $search_terms,
189
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
190
            ]
191
        };
192
    }
193
163
    my $database = Koha::Database->new();
194
    my $database = Koha::Database->new();
164
    my $schema   = $database->schema();
195
    my $schema   = $database->schema();
165
    my $rs =
196
    my $rs =
166
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
197
      $schema->resultset('AuthHeader')->search(
198
        $search_terms,
167
        { columns => [qw/ authid authtypecode marcxml /] } );
199
        { columns => [qw/ authid authtypecode marcxml /] } );
168
    my $next_func = sub {
200
    my $next_func = sub {
169
        my $row = $rs->next();
201
        my $row = $rs->next();
(-)a/misc/search_tools/rebuild_elastic_search.pl (-14 / +29 lines)
Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-s|--slice>
68
69
Only index a slice of the records. A slice is defined as two integers separated
70
by a comma. The first one is the slice to process and the second one is total
71
number of slices. e.g. --slice=1,3 processes every third record starting from
72
the first and --slice=2,3 processes every third record starting from the
73
second.
74
67
=item B<-v|--verbose>
75
=item B<-v|--verbose>
68
76
69
By default, this program only emits warnings and errors. This makes it talk
77
By default, this program only emits warnings and errors. This makes it talk
Lines 94-114 use Pod::Usage; Link Here
94
102
95
my $verbose = 0;
103
my $verbose = 0;
96
my $commit = 5000;
104
my $commit = 5000;
97
my ($delete, $help, $man);
105
my ($delete, $help, $man, $slice);
98
my ($index_biblios, $index_authorities);
106
my ($index_biblios, $index_authorities);
99
my (@biblionumbers);
107
my (@biblionumbers);
100
108
101
$|=1; # flushes output
109
$|=1; # flushes output
102
110
103
GetOptions(
111
GetOptions(
104
    'c|commit=i'       => \$commit,
112
    'c|commit=i'    => \$commit,
105
    'd|delete'         => \$delete,
113
    'd|delete'      => \$delete,
106
    'a|authorities' => \$index_authorities,
114
    'a|authorities' => \$index_authorities,
107
    'b|biblios' => \$index_biblios,
115
    'b|biblios'     => \$index_biblios,
108
    'bn|bnumber=i' => \@biblionumbers,
116
    'bn|bnumber=i'  => \@biblionumbers,
109
    'v|verbose+'       => \$verbose,
117
    's|slice=s'     => \$slice,
110
    'h|help'           => \$help,
118
    'v|verbose+'    => \$verbose,
111
    'man'              => \$man,
119
    'h|help'        => \$help,
120
    'man'           => \$man,
112
);
121
);
113
122
114
# Default is to do both
123
# Default is to do both
Lines 121-126 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here
121
130
122
sanity_check();
131
sanity_check();
123
132
133
my %iterator_options;
134
if ($slice) {
135
    my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/;
136
    _log(1, "Processing slice $slice_index of $slice_count\n");
137
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
138
}
139
124
my $next;
140
my $next;
125
if ($index_biblios) {
141
if ($index_biblios) {
126
    _log(1, "Indexing biblios\n");
142
    _log(1, "Indexing biblios\n");
Lines 131-137 if ($index_biblios) { Link Here
131
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
147
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132
        };
148
        };
133
    } else {
149
    } else {
134
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
150
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135
        $next = sub {
151
        $next = sub {
136
            $records->next();
152
            $records->next();
137
        }
153
        }
Lines 148-154 if ($index_authorities) { Link Here
148
            return ($r, $a->record);
164
            return ($r, $a->record);
149
        };
165
        };
150
    } else {
166
    } else {
151
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
167
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152
        $next = sub {
168
        $next = sub {
153
            $records->next();
169
            $records->next();
154
        }
170
        }
Lines 191-202 sub do_reindex { Link Here
191
        push @id_buffer,     $id;
207
        push @id_buffer,     $id;
192
        push @commit_buffer, $record;
208
        push @commit_buffer, $record;
193
        if ( !( --$commit_count ) ) {
209
        if ( !( --$commit_count ) ) {
194
            _log( 1, "Committing $commit records..." );
210
            _log( 1, "Committing $commit records...\n" );
195
            $indexer->update_index( \@id_buffer, \@commit_buffer );
211
            $indexer->update_index( \@id_buffer, \@commit_buffer );
196
            $commit_count  = $commit;
212
            $commit_count  = $commit;
197
            @id_buffer     = ();
213
            @id_buffer     = ();
198
            @commit_buffer = ();
214
            @commit_buffer = ();
199
            _log( 1, " done\n" );
215
            _log( 1, "Commit complete\n" );
200
        }
216
        }
201
    }
217
    }
202
218
Lines 222-226 sub sanity_check { Link Here
222
sub _log {
238
sub _log {
223
    my ($level, $msg) = @_;
239
    my ($level, $msg) = @_;
224
240
225
    print $msg if ($verbose >= $level);
241
    print "[$$] $msg" if ($verbose >= $level);
226
}
242
}
227
- 

Return to bug 21872