@@ -, +, @@ --- Koha/BiblioUtils.pm | 29 ++++++++- Koha/MetadataRecord/Authority.pm | 91 +++++++++++++++++++---------- misc/search_tools/rebuild_elastic_search.pl | 42 ++++++++----- 3 files changed, 116 insertions(+), 46 deletions(-) --- a/Koha/BiblioUtils.pm +++ a/Koha/BiblioUtils.pm @@ -101,29 +101,54 @@ sub get_from_biblionumber { =head2 get_all_biblios_iterator - my $it = Koha::BiblioUtils->get_all_biblios_iterator(); + my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::BiblioUtils of each biblio. This will include the item data. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_biblios_iterator { + my ($self, %options) = @_; + + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('Biblio')->search( {}, { columns => [qw/ biblionumber /] } ); + my $next_func = sub { - # Warn and skip bad records, otherwise we break the loop while (1) { my $row = $rs->next(); return if !$row; + # Check if biblionumber matches our slice definition + if (defined $slice_count) { + next unless $row->biblionumber % $slice_count == $slice_modulo; + } my $marc = C4::Biblio::GetMarcBiblio({ biblionumber => $row->biblionumber, embed_items => 1 }); + # Warn and skip bad records, otherwise we break the loop my $next = eval { __PACKAGE__->new($marc, $row->biblionumber); }; --- a/Koha/MetadataRecord/Authority.pm +++ a/Koha/MetadataRecord/Authority.pm @@ -150,52 +150,81 @@ sub authorized_heading { =head2 get_all_authorities_iterator - my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::MetadataRecord::Authority of each authority. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_authorities_iterator { + my ($self, %options) = @_; + + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } }, { columns => [qw/ authid authtypecode marcxml /] } ); + my $next_func = sub { - my $row = $rs->next(); - return if !$row; - my $authid = $row->authid; - my $authtypecode = $row->authtypecode; - my $marcxml = $row->marcxml; - - my $record = eval { - MARC::Record->new_from_xml( - StripNonXmlChars($marcxml), - 'UTF-8', - ( - C4::Context->preference("marcflavour") eq "UNIMARC" - ? "UNIMARCAUTH" - : C4::Context->preference("marcflavour") - ) - ); - }; - confess $@ if ($@); - $record->encoding('UTF-8'); - - # I'm not sure why we don't use the authtypecode from the database, - # but this is how the original code does it. - require C4::AuthoritiesMarc; - $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record); - - my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } ); - - return $auth; - }; - return Koha::MetadataIterator->new($next_func); + while (1) { + my $row = $rs->next(); + return if !$row; + + # Check if authid matches our slice definition + if (defined $slice_count) { + next unless $row->authid % $slice_count == $slice_modulo; + } + + my $authid = $row->authid; + my $authtypecode = $row->authtypecode; + my $marcxml = $row->marcxml; + + my $record = eval { + MARC::Record->new_from_xml( + StripNonXmlChars($marcxml), + 'UTF-8', + ( + C4::Context->preference("marcflavour") eq "UNIMARC" + ? "UNIMARCAUTH" + : C4::Context->preference("marcflavour") + ) + ); + }; + confess $@ if ($@); + $record->encoding('UTF-8'); + + # I'm not sure why we don't use the authtypecode from the database, + # but this is how the original code does it. + require C4::AuthoritiesMarc; + $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record); + + my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } ); + + return $auth; + } + }; + return Koha::MetadataIterator->new($next_func); } 1; --- a/misc/search_tools/rebuild_elastic_search.pl +++ a/misc/search_tools/rebuild_elastic_search.pl @@ -64,6 +64,14 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<-s|--slice> + +Only index a slice of the records. A slice is defined as two integers separated +by a comma. The first one is the slice to process and the second one is total +number of slices. e.g. --slice=1,3 processes every third record starting from +the first and --slice=2,3 processes every third record starting from the +second. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -94,21 +102,22 @@ use Pod::Usage; my $verbose = 0; my $commit = 5000; -my ($delete, $help, $man); +my ($delete, $help, $man, $slice); my ($index_biblios, $index_authorities); my (@biblionumbers); $|=1; # flushes output GetOptions( - 'c|commit=i' => \$commit, - 'd|delete' => \$delete, + 'c|commit=i' => \$commit, + 'd|delete' => \$delete, 'a|authorities' => \$index_authorities, - 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, - 'v|verbose+' => \$verbose, - 'h|help' => \$help, - 'man' => \$man, + 'b|biblios' => \$index_biblios, + 'bn|bnumber=i' => \@biblionumbers, + 's|slice=s' => \$slice, + 'v|verbose+' => \$verbose, + 'h|help' => \$help, + 'man' => \$man, ); # Default is to do both @@ -121,6 +130,13 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; sanity_check(); +my %iterator_options; +if ($slice) { + my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/; + _log(1, "Processing slice $slice_index of $slice_count\n"); + $iterator_options{slice} = { index => $slice_index, count => $slice_count }; +} + my $next; if ($index_biblios) { _log(1, "Indexing biblios\n"); @@ -131,7 +147,7 @@ if ($index_biblios) { return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); $next = sub { $records->next(); } @@ -148,7 +164,7 @@ if ($index_authorities) { return ($r, $a->record); }; } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); $next = sub { $records->next(); } @@ -191,12 +207,12 @@ sub do_reindex { push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 1, "Committing $commit records..." ); + _log( 1, "Committing $commit records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); - _log( 1, " done\n" ); + _log( 1, "Commit complete\n" ); } } @@ -222,5 +238,5 @@ sub sanity_check { sub _log { my ($level, $msg) = @_; - print $msg if ($verbose >= $level); + print "[$$] $msg" if ($verbose >= $level); } --