Attachment #89109 for bug #21872

View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All





=head2 get_all_biblios_iterator

    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::BiblioUtils of each biblio. This will include the item data.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=back

=cut

sub get_all_biblios_iterator {
    my ($self, %options) = @_;

    my $search_terms = {};
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('Biblio')->search(
        $search_terms,
        { columns => [qw/ biblionumber /] } );
    my $next_func = sub {
        # Warn and skip bad records, otherwise we break the loop





=head2 get_all_authorities_iterator

    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::MetadataRecord::Authority of each authority.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=back

=cut

sub get_all_authorities_iterator {
    my ($self, %options) = @_;

    my $search_terms = {
        marcxml => { '!=', undef }
    };
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
        $search_terms = {
            '-and' => [
                $search_terms,
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
            ]
        };
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('AuthHeader')->search(
        $search_terms,
        { columns => [qw/ authid authtypecode marcxml /] } );
    my $next_func = sub {
        my $row = $rs->next();

Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-33 / +111 lines)
64	repeated. This also applies to authorities via authid, so if you're using it,	64	repeated. This also applies to authorities via authid, so if you're using it,
65	you probably only want to do one or the other at a time.	65	you probably only want to do one or the other at a time.
66		66
		67	=item B<-p\|--processes>
		68
		69	Number of processes to use for indexing. This can be used to do more indexing
		70	work in parallel on multicore systems. By default, a single process is used.
		71
67	=item B<-v\|--verbose>	72	=item B<-v\|--verbose>
68		73
69	By default, this program only emits warnings and errors. This makes it talk	74	By default, this program only emits warnings and errors. This makes it talk
Lines 79-84 Full documentation. Link Here
79		84
80	=back	85	=back
81		86
		87	=head1 IMPLEMENTATION
		88
82	=cut	89	=cut
83		90
84	use autodie;	91	use autodie;
Lines 95-115 use Pod::Usage; Link Here
95		102
96	my $verbose = 0;	103	my $verbose = 0;
97	my $commit = 5000;	104	my $commit = 5000;
98	my ($delete, $help, $man);	105	my ($delete, $help, $man, $processes);
99	my ($index_biblios, $index_authorities);	106	my ($index_biblios, $index_authorities);
100	my (@biblionumbers);	107	my (@record_numbers);
101		108
102	$\|=1; # flushes output	109	$\|=1; # flushes output
103		110
104	GetOptions(	111	GetOptions(
105	'c\|commit=i' => \$commit,	112	'c\|commit=i' => \$commit,
106	'd\|delete' => \$delete,	113	'd\|delete' => \$delete,
107	'a\|authorities' => \$index_authorities,	114	'a\|authorities' => \$index_authorities,
108	'b\|biblios' => \$index_biblios,	115	'b\|biblios' => \$index_biblios,
109	'bn\|bnumber=i' => \@biblionumbers,	116	'bn\|bnumber=i' => \@record_numbers,
110	'v\|verbose+' => \$verbose,	117	'p\|processes=i' => \$processes,
111	'h\|help' => \$help,	118	'v\|verbose+' => \$verbose,
112	'man' => \$man,	119	'h\|help' => \$help,
		120	'man' => \$man,
113	);	121	);
114		122
115	# Default is to do both	123	# Default is to do both
Lines 120-168 unless ($index_authorities \|\| $index_biblios) { Link Here
120	pod2usage(1) if $help;	128	pod2usage(1) if $help;
121	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;	129	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
122		130
123	sanity_check();	131	_sanity_check();
		132
		133	_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
		134	_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
		135
		136	my $slice_index = 0;
		137	my $slice_count = $processes // 1;
		138
		139	if ($slice_count > 1) {
		140	# Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
		141	$slice_index = 1;
		142	for (my $proc = 2; $proc <= $processes; $proc++) {
		143	my $pid = fork();
		144	die "Failed to fork a child process\n" unless defined $pid;
		145	if ($pid == 0) {
		146	# Child process, give it a slice to process
		147	$slice_index = $proc;
		148	last;
		149	}
		150	}
		151	# Fudge the commit count a bit to spread out the Elasticsearch commits
		152	$commit = 1 + 0.10 ($slice_index - 1);
		153	}
		154
		155	my %iterator_options;
		156	if ($slice_index) {
		157	_log(1, "Processing slice $slice_index of $slice_count\n");
		158	$iterator_options{slice} = { index => $slice_index, count => $slice_count };
		159	}
124		160
125	my $next;	161	my $next;
126	if ($index_biblios) {	162	if ($index_biblios) {
127	_log(1, "Indexing biblios\n");	163	_log(1, "Indexing biblios\n");
128	if (@biblionumbers) {	164	if (@record_numbers) {
129	$next = sub {	165	$next = sub {
130	my $r = shift @biblionumbers;	166	my $r = shift @record_numbers;
131	return () unless defined $r;	167	return () unless defined $r;
132	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));	168	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
133	};	169	};
134	} else {	170	} else {
135	my $records = Koha::BiblioUtils->get_all_biblios_iterator();	171	my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
136	$next = sub {	172	$next = sub {
137	$records->next();	173	$records->next();
138	}	174	}
139	}	175	}
140	do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);	176	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
141	}	177	}
142	if ($index_authorities) {	178	if ($index_authorities) {
143	_log(1, "Indexing authorities\n");	179	_log(1, "Indexing authorities\n");
144	if (@biblionumbers) {	180	if (@record_numbers) {
145	$next = sub {	181	$next = sub {
146	my $r = shift @biblionumbers;	182	my $r = shift @record_numbers;
147	return () unless defined $r;	183	return () unless defined $r;
148	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);	184	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
149	return ($r, $a->record);	185	return ($r, $a->record);
150	};	186	};
151	} else {	187	} else {
152	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	188	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
153	$next = sub {	189	$next = sub {
154	$records->next();	190	$records->next();
155	}	191	}
156	}	192	}
157	do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);	193	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
158	}	194	}
159		195
160	sub do_reindex {	196	if ($slice_index == 1) {
161	my ( $next, $index_name ) = @_;	197	# Main process, wait for children
		198	for (my $proc = 2; $proc <= $processes; $proc++) {
		199	wait();
		200	}
		201	}
		202
		203	=head2 _verify_index_state
		204
		205	_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
		206
		207	Checks the index state and recreates it if requested.
162		208
		209	=cut
		210
		211	sub _verify_index_state {
		212	my ( $index_name, $recreate ) = @_;
		213
		214	_log(1, "Checking state of $index_name index\n");
163	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );	215	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
164		216
165	if ($delete) {	217	if ($recreate) {
		218	_log(1, "Dropping and recreating $index_name index\n");
166	$indexer->drop_index() if $indexer->index_exists();	219	$indexer->drop_index() if $indexer->index_exists();
167	$indexer->create_index();	220	$indexer->create_index();
168	}	221	}
Lines 175-180 sub do_reindex { Link Here
175	} elsif ($indexer->is_index_status_recreate_required) {	228	} elsif ($indexer->is_index_status_recreate_required) {
176	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;	229	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
177	}	230	}
		231	}
		232
		233	=head2 _do_reindex
		234
		235	_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
		236
		237	Does the actual reindexing. $callback is a function that always returns the next record.
		238
		239	=cut
		240
		241	sub _do_reindex {
		242	my ( $next, $index_name ) = @_;
		243
		244	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
178		245
179	my $count = 0;	246	my $count = 0;
180	my $commit_count = $commit;	247	my $commit_count = $commit;
Lines 192-203 sub do_reindex { Link Here
192	push @id_buffer, $id;	259	push @id_buffer, $id;
193	push @commit_buffer, $record;	260	push @commit_buffer, $record;
194	if ( !( --$commit_count ) ) {	261	if ( !( --$commit_count ) ) {
195	_log( 1, "Committing $commit records..." );	262	_log( 1, "Committing $commit records...\n" );
196	$indexer->update_index( \@id_buffer, \@commit_buffer );	263	$indexer->update_index( \@id_buffer, \@commit_buffer );
197	$commit_count = $commit;	264	$commit_count = $commit;
198	@id_buffer = ();	265	@id_buffer = ();
199	@commit_buffer = ();	266	@commit_buffer = ();
200	_log( 1, " done\n" );	267	_log( 1, "Commit complete\n" );
201	}	268	}
202	}	269	}
203		270
Lines 207-227 sub do_reindex { Link Here
207	_log( 1, "Total $count records indexed\n" );	274	_log( 1, "Total $count records indexed\n" );
208	}	275	}
209		276
210	# Checks some basic stuff to ensure that it's sane before we start.	277	=head2 _sanity_check
211	sub sanity_check {	278
		279	_sanity_check();
		280
		281	Checks some basic stuff to ensure that it's sane before we start.
		282
		283	=cut
		284
		285	sub _sanity_check {
212	# Do we have an elasticsearch block defined?	286	# Do we have an elasticsearch block defined?
213	my $conf = C4::Context->config('elasticsearch');	287	my $conf = C4::Context->config('elasticsearch');
214	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );	288	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
215	}	289	}
216		290
217	# Output progress information.	291	=head2 _log
218	#	292
219	# _log($level, $msg);	293	_log($level, "Message\n");
220	#	294
221	# Will output $msg if the verbosity setting is set to $level or more. Will	295	Output progress information.
222	# not include a trailing newline.	296
		297	Will output the message if verbosity level is set to $level or more. Will not
		298	include a trailing newline automatically.
		299
		300	=cut
		301
223	sub _log {	302	sub _log {
224	my ($level, $msg) = @_;	303	my ($level, $msg) = @_;
225		304
226	print $msg if ($verbose >= $level);	305	print "[$$] $msg" if ($verbose >= $level);
227	}	306	}
228	-

Return to bug 21872

Lines 101-120 sub get_from_biblionumber { Link Here

(-)a/Koha/BiblioUtils.pm (-2 / +28 lines)
101		101
102	=head2 get_all_biblios_iterator	102	=head2 get_all_biblios_iterator
103		103
104	my $it = Koha::BiblioUtils->get_all_biblios_iterator();	104	my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105		105
106	This will provide an iterator object that will, one by one, provide the	106	This will provide an iterator object that will, one by one, provide the
107	Koha::BiblioUtils of each biblio. This will include the item data.	107	Koha::BiblioUtils of each biblio. This will include the item data.
108		108
109	The iterator is a Koha::MetadataIterator object.	109	The iterator is a Koha::MetadataIterator object.
110		110
		111	Possible options are:
		112
		113	=over 4
		114
		115	=item C<slice>
		116
		117	slice may be defined as a hash of two values: index and count. index
		118	is the slice number to process and count is total number of slices.
		119	With this information the iterator returns just the given slice of
		120	records instead of all.
		121
		122	=back
		123
111	=cut	124	=cut
112		125
113	sub get_all_biblios_iterator {	126	sub get_all_biblios_iterator {
		127	my ($self, %options) = @_;
		128
		129	my $search_terms = {};
		130	my ($slice_modulo, $slice_count);
		131	if ($options{slice}) {
		132	$slice_count = $options{slice}->{count};
		133	$slice_modulo = $options{slice}->{index};
		134	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		135
		136	$search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
		137	}
		138
114	my $database = Koha::Database->new();	139	my $database = Koha::Database->new();
115	my $schema = $database->schema();	140	my $schema = $database->schema();
116	my $rs =	141	my $rs =
117	$schema->resultset('Biblio')->search( {},	142	$schema->resultset('Biblio')->search(
		143	$search_terms,
118	{ columns => [qw/ biblionumber /] } );	144	{ columns => [qw/ biblionumber /] } );
119	my $next_func = sub {	145	my $next_func = sub {
120	# Warn and skip bad records, otherwise we break the loop	146	# Warn and skip bad records, otherwise we break the loop

Lines 151-170 sub authorized_heading { Link Here

(-)a/Koha/MetadataRecord/Authority.pm (-2 / +36 lines)
151		151
152	=head2 get_all_authorities_iterator	152	=head2 get_all_authorities_iterator
153		153
154	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	154	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
155		155
156	This will provide an iterator object that will, one by one, provide the	156	This will provide an iterator object that will, one by one, provide the
157	Koha::MetadataRecord::Authority of each authority.	157	Koha::MetadataRecord::Authority of each authority.
158		158
159	The iterator is a Koha::MetadataIterator object.	159	The iterator is a Koha::MetadataIterator object.
160		160
		161	Possible options are:
		162
		163	=over 4
		164
		165	=item C<slice>
		166
		167	slice may be defined as a hash of two values: index and count. index
		168	is the slice number to process and count is total number of slices.
		169	With this information the iterator returns just the given slice of
		170	records instead of all.
		171
		172	=back
		173
161	=cut	174	=cut
162		175
163	sub get_all_authorities_iterator {	176	sub get_all_authorities_iterator {
		177	my ($self, %options) = @_;
		178
		179	my $search_terms = {
		180	marcxml => { '!=', undef }
		181	};
		182	my ($slice_modulo, $slice_count);
		183	if ($options{slice}) {
		184	$slice_count = $options{slice}->{count};
		185	$slice_modulo = $options{slice}->{index};
		186	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		187
		188	$search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
		189	$search_terms = {
		190	'-and' => [
		191	$search_terms,
		192	\[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
		193	]
		194	};
		195	}
		196
164	my $database = Koha::Database->new();	197	my $database = Koha::Database->new();
165	my $schema = $database->schema();	198	my $schema = $database->schema();
166	my $rs =	199	my $rs =
167	$schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },	200	$schema->resultset('AuthHeader')->search(
		201	$search_terms,
168	{ columns => [qw/ authid authtypecode marcxml /] } );	202	{ columns => [qw/ authid authtypecode marcxml /] } );
169	my $next_func = sub {	203	my $next_func = sub {
170	my $row = $rs->next();	204	my $row = $rs->next();