Attachment #82599 for bug #21872

View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All





=head2 get_all_biblios_iterator

    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::BiblioUtils of each biblio. This will include the item data.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=cut

sub get_all_biblios_iterator {
    my ($self, %options) = @_;

    my $search_terms = {};
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('Biblio')->search(
        $search_terms,
        { columns => [qw/ biblionumber /] } );
    my $next_func = sub {
        # Warn and skip bad records, otherwise we break the loop





=head2 get_all_authorities_iterator

    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::MetadataRecord::Authority of each authority.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=cut

sub get_all_authorities_iterator {
    my ($self, %options) = @_;

    my $search_terms = {
        marcxml => { '!=', undef }
    };
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
        $search_terms = {
            '-and' => [
                $search_terms,
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
            ]
        };
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('AuthHeader')->search(
        $search_terms,
        { columns => [qw/ authid authtypecode marcxml /] } );
    my $next_func = sub {
        my $row = $rs->next();




repeated. This also applies to authorities via authid, so if you're using it,
you probably only want to do one or the other at a time.

=item B<-s|--slice>

Only index a slice of the records. A slice is defined as two integers separated
by a comma. The first one is the slice to process and the second one is total
number of slices. e.g. --slice=1,3 processes every third record starting from
the first and --slice=2,3 processes every third record starting from the
second.

=item B<-v|--verbose>

By default, this program only emits warnings and errors. This makes it talk


my $verbose = 0;
my $commit = 5000;
my ($delete, $help, $man, $slice);
my ($index_biblios, $index_authorities);
my (@biblionumbers);

$|=1; # flushes output

GetOptions(
    'c|commit=i'    => \$commit,
    'd|delete'      => \$delete,
    'a|authorities' => \$index_authorities,
    'b|biblios'     => \$index_biblios,
    'bn|bnumber=i'  => \@biblionumbers,
    's|slice=s'     => \$slice,
    'v|verbose+'    => \$verbose,
    'h|help'        => \$help,
    'man'           => \$man,
);

# Default is to do both


sanity_check();

my %iterator_options;
if ($slice) {
    my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/;
    _log(1, "Processing slice $slice_index of $slice_count\n");
    $iterator_options{slice} = { index => $slice_index, count => $slice_count };
}

my $next;
if ($index_biblios) {
    _log(1, "Indexing biblios\n");

            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
        };
    } else {
        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
        $next = sub {
            $records->next();
        }

            return ($r, $a->record);
        };
    } else {
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
        $next = sub {
            $records->next();
        }

        push @id_buffer,     $id;
        push @commit_buffer, $record;
        if ( !( --$commit_count ) ) {
            _log( 1, "Committing $commit records...\n" );
            $indexer->update_index( \@id_buffer, \@commit_buffer );
            $commit_count  = $commit;
            @id_buffer     = ();
            @commit_buffer = ();
            _log( 1, "Commit complete\n" );
        }
    }


sub _log {
    my ($level, $msg) = @_;

    print "[$$] $msg" if ($verbose >= $level);
}
- 

Return to bug 21872

Lines 101-120 sub get_from_biblionumber { Link Here

(-)a/Koha/BiblioUtils.pm (-2 / +26 lines)
101		101
102	=head2 get_all_biblios_iterator	102	=head2 get_all_biblios_iterator
103		103
104	my $it = Koha::BiblioUtils->get_all_biblios_iterator();	104	my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105		105
106	This will provide an iterator object that will, one by one, provide the	106	This will provide an iterator object that will, one by one, provide the
107	Koha::BiblioUtils of each biblio. This will include the item data.	107	Koha::BiblioUtils of each biblio. This will include the item data.
108		108
109	The iterator is a Koha::MetadataIterator object.	109	The iterator is a Koha::MetadataIterator object.
110		110
		111	Possible options are:
		112
		113	=over 4
		114
		115	=item C<slice>
		116
		117	slice may be defined as a hash of two values: index and count. index
		118	is the slice number to process and count is total number of slices.
		119	With this information the iterator returns just the given slice of
		120	records instead of all.
		121
111	=cut	122	=cut
112		123
113	sub get_all_biblios_iterator {	124	sub get_all_biblios_iterator {
		125	my ($self, %options) = @_;
		126
		127	my $search_terms = {};
		128	my ($slice_modulo, $slice_count);
		129	if ($options{slice}) {
		130	$slice_count = $options{slice}->{count};
		131	$slice_modulo = $options{slice}->{index};
		132	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		133
		134	$search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
		135	}
		136
114	my $database = Koha::Database->new();	137	my $database = Koha::Database->new();
115	my $schema = $database->schema();	138	my $schema = $database->schema();
116	my $rs =	139	my $rs =
117	$schema->resultset('Biblio')->search( {},	140	$schema->resultset('Biblio')->search(
		141	$search_terms,
118	{ columns => [qw/ biblionumber /] } );	142	{ columns => [qw/ biblionumber /] } );
119	my $next_func = sub {	143	my $next_func = sub {
120	# Warn and skip bad records, otherwise we break the loop	144	# Warn and skip bad records, otherwise we break the loop

Lines 150-169 sub authorized_heading { Link Here

(-)a/Koha/MetadataRecord/Authority.pm (-2 / +34 lines)
150		150
151	=head2 get_all_authorities_iterator	151	=head2 get_all_authorities_iterator
152		152
153	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	153	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154		154
155	This will provide an iterator object that will, one by one, provide the	155	This will provide an iterator object that will, one by one, provide the
156	Koha::MetadataRecord::Authority of each authority.	156	Koha::MetadataRecord::Authority of each authority.
157		157
158	The iterator is a Koha::MetadataIterator object.	158	The iterator is a Koha::MetadataIterator object.
159		159
		160	Possible options are:
		161
		162	=over 4
		163
		164	=item C<slice>
		165
		166	slice may be defined as a hash of two values: index and count. index
		167	is the slice number to process and count is total number of slices.
		168	With this information the iterator returns just the given slice of
		169	records instead of all.
		170
160	=cut	171	=cut
161		172
162	sub get_all_authorities_iterator {	173	sub get_all_authorities_iterator {
		174	my ($self, %options) = @_;
		175
		176	my $search_terms = {
		177	marcxml => { '!=', undef }
		178	};
		179	my ($slice_modulo, $slice_count);
		180	if ($options{slice}) {
		181	$slice_count = $options{slice}->{count};
		182	$slice_modulo = $options{slice}->{index};
		183	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		184
		185	$search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
		186	$search_terms = {
		187	'-and' => [
		188	$search_terms,
		189	\[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
		190	]
		191	};
		192	}
		193
163	my $database = Koha::Database->new();	194	my $database = Koha::Database->new();
164	my $schema = $database->schema();	195	my $schema = $database->schema();
165	my $rs =	196	my $rs =
166	$schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },	197	$schema->resultset('AuthHeader')->search(
		198	$search_terms,
167	{ columns => [qw/ authid authtypecode marcxml /] } );	199	{ columns => [qw/ authid authtypecode marcxml /] } );
168	my $next_func = sub {	200	my $next_func = sub {
169	my $row = $rs->next();	201	my $row = $rs->next();

Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-14 / +29 lines)
64	repeated. This also applies to authorities via authid, so if you're using it,	64	repeated. This also applies to authorities via authid, so if you're using it,
65	you probably only want to do one or the other at a time.	65	you probably only want to do one or the other at a time.
66		66
		67	=item B<-s\|--slice>
		68
		69	Only index a slice of the records. A slice is defined as two integers separated
		70	by a comma. The first one is the slice to process and the second one is total
		71	number of slices. e.g. --slice=1,3 processes every third record starting from
		72	the first and --slice=2,3 processes every third record starting from the
		73	second.
		74
67	=item B<-v\|--verbose>	75	=item B<-v\|--verbose>
68		76
69	By default, this program only emits warnings and errors. This makes it talk	77	By default, this program only emits warnings and errors. This makes it talk
Lines 94-114 use Pod::Usage; Link Here
94		102
95	my $verbose = 0;	103	my $verbose = 0;
96	my $commit = 5000;	104	my $commit = 5000;
97	my ($delete, $help, $man);	105	my ($delete, $help, $man, $slice);
98	my ($index_biblios, $index_authorities);	106	my ($index_biblios, $index_authorities);
99	my (@biblionumbers);	107	my (@biblionumbers);
100		108
101	$\|=1; # flushes output	109	$\|=1; # flushes output
102		110
103	GetOptions(	111	GetOptions(
104	'c\|commit=i' => \$commit,	112	'c\|commit=i' => \$commit,
105	'd\|delete' => \$delete,	113	'd\|delete' => \$delete,
106	'a\|authorities' => \$index_authorities,	114	'a\|authorities' => \$index_authorities,
107	'b\|biblios' => \$index_biblios,	115	'b\|biblios' => \$index_biblios,
108	'bn\|bnumber=i' => \@biblionumbers,	116	'bn\|bnumber=i' => \@biblionumbers,
109	'v\|verbose+' => \$verbose,	117	's\|slice=s' => \$slice,
110	'h\|help' => \$help,	118	'v\|verbose+' => \$verbose,
111	'man' => \$man,	119	'h\|help' => \$help,
		120	'man' => \$man,
112	);	121	);
113		122
114	# Default is to do both	123	# Default is to do both
Lines 121-126 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here
121		130
122	sanity_check();	131	sanity_check();
123		132
		133	my %iterator_options;
		134	if ($slice) {
		135	my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/;
		136	_log(1, "Processing slice $slice_index of $slice_count\n");
		137	$iterator_options{slice} = { index => $slice_index, count => $slice_count };
		138	}
		139
124	my $next;	140	my $next;
125	if ($index_biblios) {	141	if ($index_biblios) {
126	_log(1, "Indexing biblios\n");	142	_log(1, "Indexing biblios\n");
Lines 131-137 if ($index_biblios) { Link Here
131	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));	147	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132	};	148	};
133	} else {	149	} else {
134	my $records = Koha::BiblioUtils->get_all_biblios_iterator();	150	my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135	$next = sub {	151	$next = sub {
136	$records->next();	152	$records->next();
137	}	153	}
Lines 148-154 if ($index_authorities) { Link Here
148	return ($r, $a->record);	164	return ($r, $a->record);
149	};	165	};
150	} else {	166	} else {
151	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	167	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152	$next = sub {	168	$next = sub {
153	$records->next();	169	$records->next();
154	}	170	}
Lines 191-202 sub do_reindex { Link Here
191	push @id_buffer, $id;	207	push @id_buffer, $id;
192	push @commit_buffer, $record;	208	push @commit_buffer, $record;
193	if ( !( --$commit_count ) ) {	209	if ( !( --$commit_count ) ) {
194	_log( 1, "Committing $commit records..." );	210	_log( 1, "Committing $commit records...\n" );
195	$indexer->update_index( \@id_buffer, \@commit_buffer );	211	$indexer->update_index( \@id_buffer, \@commit_buffer );
196	$commit_count = $commit;	212	$commit_count = $commit;
197	@id_buffer = ();	213	@id_buffer = ();
198	@commit_buffer = ();	214	@commit_buffer = ();
199	_log( 1, " done\n" );	215	_log( 1, "Commit complete\n" );
200	}	216	}
201	}	217	}
202		218
Lines 222-226 sub sanity_check { Link Here
222	sub _log {	238	sub _log {
223	my ($level, $msg) = @_;	239	my ($level, $msg) = @_;
224		240
225	print $msg if ($verbose >= $level);	241	print "[$$] $msg" if ($verbose >= $level);
226	}	242	}
227	-