View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/Koha/BiblioUtils.pm (-2 / +27 lines)
Lines 101-129 sub get_from_biblionumber { Link Here
101
101
102
=head2 get_all_biblios_iterator
102
=head2 get_all_biblios_iterator
103
103
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator();
104
    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105
105
106
This will provide an iterator object that will, one by one, provide the
106
This will provide an iterator object that will, one by one, provide the
107
Koha::BiblioUtils of each biblio. This will include the item data.
107
Koha::BiblioUtils of each biblio. This will include the item data.
108
108
109
The iterator is a Koha::MetadataIterator object.
109
The iterator is a Koha::MetadataIterator object.
110
110
111
Possible options are:
112
113
=over 4
114
115
=item C<slice>
116
117
slice may be defined as a hash of two values: index and count. index
118
is the slice number to process and count is total number of slices. 
119
With this information the iterator returns just the given slice of
120
records instead of all.
121
111
=cut
122
=cut
112
123
113
sub get_all_biblios_iterator {
124
sub get_all_biblios_iterator {
125
    my ($self, %options) = @_;
126
127
    my ($slice_modulo, $slice_count);
128
    if ($options{slice}) {
129
        $slice_count = $options{slice}->{count};
130
        $slice_modulo = $options{slice}->{index};
131
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
132
    }
133
114
    my $database = Koha::Database->new();
134
    my $database = Koha::Database->new();
115
    my $schema   = $database->schema();
135
    my $schema   = $database->schema();
116
    my $rs =
136
    my $rs =
117
      $schema->resultset('Biblio')->search( {},
137
      $schema->resultset('Biblio')->search( {},
118
        { columns => [qw/ biblionumber /] } );
138
        { columns => [qw/ biblionumber /] } );
139
119
    my $next_func = sub {
140
    my $next_func = sub {
120
        # Warn and skip bad records, otherwise we break the loop
121
        while (1) {
141
        while (1) {
122
            my $row = $rs->next();
142
            my $row = $rs->next();
123
            return if !$row;
143
            return if !$row;
144
            # Check if biblionumber matches our slice definition
145
            if (defined $slice_count) {
146
                next unless $row->biblionumber % $slice_count == $slice_modulo;
147
            }
124
            my $marc = C4::Biblio::GetMarcBiblio({
148
            my $marc = C4::Biblio::GetMarcBiblio({
125
                biblionumber => $row->biblionumber,
149
                biblionumber => $row->biblionumber,
126
                embed_items  => 1 });
150
                embed_items  => 1 });
151
            # Warn and skip bad records, otherwise we break the loop
127
            my $next = eval {
152
            my $next = eval {
128
                __PACKAGE__->new($marc, $row->biblionumber);
153
                __PACKAGE__->new($marc, $row->biblionumber);
129
            };
154
            };
(-)a/Koha/MetadataRecord/Authority.pm (-31 / +60 lines)
Lines 150-201 sub authorized_heading { Link Here
150
150
151
=head2 get_all_authorities_iterator
151
=head2 get_all_authorities_iterator
152
152
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
153
    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154
154
155
This will provide an iterator object that will, one by one, provide the
155
This will provide an iterator object that will, one by one, provide the
156
Koha::MetadataRecord::Authority of each authority.
156
Koha::MetadataRecord::Authority of each authority.
157
157
158
The iterator is a Koha::MetadataIterator object.
158
The iterator is a Koha::MetadataIterator object.
159
159
160
Possible options are:
161
162
=over 4
163
164
=item C<slice>
165
166
slice may be defined as a hash of two values: index and count. index
167
is the slice number to process and count is total number of slices. 
168
With this information the iterator returns just the given slice of
169
records instead of all.
170
160
=cut
171
=cut
161
172
162
sub get_all_authorities_iterator {
173
sub get_all_authorities_iterator {
174
    my ($self, %options) = @_;
175
176
    my ($slice_modulo, $slice_count);
177
    if ($options{slice}) {
178
        $slice_count = $options{slice}->{count};
179
        $slice_modulo = $options{slice}->{index};
180
        $slice_modulo = 0 if ($slice_modulo == $slice_count);
181
    }
182
163
    my $database = Koha::Database->new();
183
    my $database = Koha::Database->new();
164
    my $schema   = $database->schema();
184
    my $schema   = $database->schema();
165
    my $rs =
185
    my $rs =
166
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
186
      $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },
167
        { columns => [qw/ authid authtypecode marcxml /] } );
187
        { columns => [qw/ authid authtypecode marcxml /] } );
188
168
    my $next_func = sub {
189
    my $next_func = sub {
169
        my $row = $rs->next();
190
        while (1) {
170
        return if !$row;
191
            my $row = $rs->next();
171
        my $authid       = $row->authid;
192
            return if !$row;
172
        my $authtypecode = $row->authtypecode;
193
173
        my $marcxml      = $row->marcxml;
194
            # Check if authid matches our slice definition
174
195
            if (defined $slice_count) {
175
        my $record = eval {
196
                next unless $row->authid % $slice_count == $slice_modulo;
176
            MARC::Record->new_from_xml(
197
            }
177
                StripNonXmlChars($marcxml),
198
178
                'UTF-8',
199
            my $authid       = $row->authid;
179
                (
200
            my $authtypecode = $row->authtypecode;
180
                    C4::Context->preference("marcflavour") eq "UNIMARC"
201
            my $marcxml      = $row->marcxml;
181
                    ? "UNIMARCAUTH"
202
182
                    : C4::Context->preference("marcflavour")
203
            my $record = eval {
183
                )
204
                MARC::Record->new_from_xml(
184
            );
205
                    StripNonXmlChars($marcxml),
185
        };
206
                    'UTF-8',
186
        confess $@ if ($@);
207
                    (
187
        $record->encoding('UTF-8');
208
                        C4::Context->preference("marcflavour") eq "UNIMARC"
188
209
                        ? "UNIMARCAUTH"
189
        # I'm not sure why we don't use the authtypecode from the database,
210
                        : C4::Context->preference("marcflavour")
190
        # but this is how the original code does it.
211
                    )
191
        require C4::AuthoritiesMarc;
212
                );
192
        $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
213
            };
193
214
            confess $@ if ($@);
194
        my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } );
215
            $record->encoding('UTF-8');
195
216
196
        return $auth;
217
            # I'm not sure why we don't use the authtypecode from the database,
197
      };
218
            # but this is how the original code does it.
198
      return Koha::MetadataIterator->new($next_func);
219
            require C4::AuthoritiesMarc;
220
            $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
221
222
            my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } );
223
224
            return $auth;
225
        }
226
    };
227
    return Koha::MetadataIterator->new($next_func);
199
}
228
}
200
229
201
1;
230
1;
(-)a/misc/search_tools/rebuild_elastic_search.pl (-14 / +29 lines)
Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-s|--slice>
68
69
Only index a slice of the records. A slice is defined as two integers separated
70
by a comma. The first one is the slice to process and the second one is total
71
number of slices. e.g. --slice=1,3 processes every third record starting from
72
the first and --slice=2,3 processes every third record starting from the 
73
second.
74
67
=item B<-v|--verbose>
75
=item B<-v|--verbose>
68
76
69
By default, this program only emits warnings and errors. This makes it talk
77
By default, this program only emits warnings and errors. This makes it talk
Lines 94-114 use Pod::Usage; Link Here
94
102
95
my $verbose = 0;
103
my $verbose = 0;
96
my $commit = 5000;
104
my $commit = 5000;
97
my ($delete, $help, $man);
105
my ($delete, $help, $man, $slice);
98
my ($index_biblios, $index_authorities);
106
my ($index_biblios, $index_authorities);
99
my (@biblionumbers);
107
my (@biblionumbers);
100
108
101
$|=1; # flushes output
109
$|=1; # flushes output
102
110
103
GetOptions(
111
GetOptions(
104
    'c|commit=i'       => \$commit,
112
    'c|commit=i'    => \$commit,
105
    'd|delete'         => \$delete,
113
    'd|delete'      => \$delete,
106
    'a|authorities' => \$index_authorities,
114
    'a|authorities' => \$index_authorities,
107
    'b|biblios' => \$index_biblios,
115
    'b|biblios'     => \$index_biblios,
108
    'bn|bnumber=i' => \@biblionumbers,
116
    'bn|bnumber=i'  => \@biblionumbers,
109
    'v|verbose+'       => \$verbose,
117
    's|slice=s'     => \$slice,
110
    'h|help'           => \$help,
118
    'v|verbose+'    => \$verbose,
111
    'man'              => \$man,
119
    'h|help'        => \$help,
120
    'man'           => \$man,
112
);
121
);
113
122
114
# Default is to do both
123
# Default is to do both
Lines 121-126 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here
121
130
122
sanity_check();
131
sanity_check();
123
132
133
my %iterator_options;
134
if ($slice) {
135
    my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/;
136
    _log(1, "Processing slice $slice_index of $slice_count\n");
137
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
138
}
139
124
my $next;
140
my $next;
125
if ($index_biblios) {
141
if ($index_biblios) {
126
    _log(1, "Indexing biblios\n");
142
    _log(1, "Indexing biblios\n");
Lines 131-137 if ($index_biblios) { Link Here
131
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
147
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132
        };
148
        };
133
    } else {
149
    } else {
134
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
150
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135
        $next = sub {
151
        $next = sub {
136
            $records->next();
152
            $records->next();
137
        }
153
        }
Lines 148-154 if ($index_authorities) { Link Here
148
            return ($r, $a->record);
164
            return ($r, $a->record);
149
        };
165
        };
150
    } else {
166
    } else {
151
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
167
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152
        $next = sub {
168
        $next = sub {
153
            $records->next();
169
            $records->next();
154
        }
170
        }
Lines 191-202 sub do_reindex { Link Here
191
        push @id_buffer,     $id;
207
        push @id_buffer,     $id;
192
        push @commit_buffer, $record;
208
        push @commit_buffer, $record;
193
        if ( !( --$commit_count ) ) {
209
        if ( !( --$commit_count ) ) {
194
            _log( 1, "Committing $commit records..." );
210
            _log( 1, "Committing $commit records...\n" );
195
            $indexer->update_index( \@id_buffer, \@commit_buffer );
211
            $indexer->update_index( \@id_buffer, \@commit_buffer );
196
            $commit_count  = $commit;
212
            $commit_count  = $commit;
197
            @id_buffer     = ();
213
            @id_buffer     = ();
198
            @commit_buffer = ();
214
            @commit_buffer = ();
199
            _log( 1, " done\n" );
215
            _log( 1, "Commit complete\n" );
200
        }
216
        }
201
    }
217
    }
202
218
Lines 222-226 sub sanity_check { Link Here
222
sub _log {
238
sub _log {
223
    my ($level, $msg) = @_;
239
    my ($level, $msg) = @_;
224
240
225
    print $msg if ($verbose >= $level);
241
    print "[$$] $msg" if ($verbose >= $level);
226
}
242
}
227
- 

Return to bug 21872