Attachment #85975 for bug #21872

View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All





=head2 get_all_biblios_iterator

    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::BiblioUtils of each biblio. This will include the item data.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=back

=cut

sub get_all_biblios_iterator {
    my ($self, %options) = @_;

    my $search_terms = {};
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('Biblio')->search(
        $search_terms,
        { columns => [qw/ biblionumber /] } );
    my $next_func = sub {
        # Warn and skip bad records, otherwise we break the loop





=head2 get_all_authorities_iterator

    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);

This will provide an iterator object that will, one by one, provide the
Koha::MetadataRecord::Authority of each authority.

The iterator is a Koha::MetadataIterator object.

Possible options are:

=over 4

=item C<slice>

slice may be defined as a hash of two values: index and count. index
is the slice number to process and count is total number of slices.
With this information the iterator returns just the given slice of
records instead of all.

=back

=cut

sub get_all_authorities_iterator {
    my ($self, %options) = @_;

    my $search_terms = {
        marcxml => { '!=', undef }
    };
    my ($slice_modulo, $slice_count);
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $slice_modulo = 0 if ($slice_modulo == $slice_count);

        $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
        $search_terms = {
            '-and' => [
                $search_terms,
                \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
            ]
        };
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('AuthHeader')->search(
        $search_terms,
        { columns => [qw/ authid authtypecode marcxml /] } );
    my $next_func = sub {
        my $row = $rs->next();

Lines 64-69 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-33 / +111 lines)
64	repeated. This also applies to authorities via authid, so if you're using it,	64	repeated. This also applies to authorities via authid, so if you're using it,
65	you probably only want to do one or the other at a time.	65	you probably only want to do one or the other at a time.
66		66
		67	=item B<-p\|--processes>
		68
		69	Number of processes to use for indexing. This can be used to do more indexing
		70	work in parallel on multicore systems. By default, a single process is used.
		71
67	=item B<-v\|--verbose>	72	=item B<-v\|--verbose>
68		73
69	By default, this program only emits warnings and errors. This makes it talk	74	By default, this program only emits warnings and errors. This makes it talk
Lines 79-84 Full documentation. Link Here
79		84
80	=back	85	=back
81		86
		87	=head1 IMPLEMENTATION
		88
82	=cut	89	=cut
83		90
84	use autodie;	91	use autodie;
Lines 94-114 use Pod::Usage; Link Here
94		101
95	my $verbose = 0;	102	my $verbose = 0;
96	my $commit = 5000;	103	my $commit = 5000;
97	my ($delete, $help, $man);	104	my ($delete, $help, $man, $processes);
98	my ($index_biblios, $index_authorities);	105	my ($index_biblios, $index_authorities);
99	my (@biblionumbers);	106	my (@record_numbers);
100		107
101	$\|=1; # flushes output	108	$\|=1; # flushes output
102		109
103	GetOptions(	110	GetOptions(
104	'c\|commit=i' => \$commit,	111	'c\|commit=i' => \$commit,
105	'd\|delete' => \$delete,	112	'd\|delete' => \$delete,
106	'a\|authorities' => \$index_authorities,	113	'a\|authorities' => \$index_authorities,
107	'b\|biblios' => \$index_biblios,	114	'b\|biblios' => \$index_biblios,
108	'bn\|bnumber=i' => \@biblionumbers,	115	'bn\|bnumber=i' => \@record_numbers,
109	'v\|verbose+' => \$verbose,	116	'p\|processes=i' => \$processes,
110	'h\|help' => \$help,	117	'v\|verbose+' => \$verbose,
111	'man' => \$man,	118	'h\|help' => \$help,
		119	'man' => \$man,
112	);	120	);
113		121
114	# Default is to do both	122	# Default is to do both
Lines 119-167 unless ($index_authorities \|\| $index_biblios) { Link Here
119	pod2usage(1) if $help;	127	pod2usage(1) if $help;
120	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;	128	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
121		129
122	sanity_check();	130	_sanity_check();
		131
		132	_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
		133	_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
		134
		135	my $slice_index = 0;
		136	my $slice_count = $processes // 1;
		137
		138	if ($slice_count > 1) {
		139	# Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
		140	$slice_index = 1;
		141	for (my $proc = 2; $proc <= $processes; $proc++) {
		142	my $pid = fork();
		143	die "Failed to fork a child process\n" unless defined $pid;
		144	if ($pid == 0) {
		145	# Child process, give it a slice to process
		146	$slice_index = $proc;
		147	last;
		148	}
		149	}
		150	# Fudge the commit count a bit to spread out the Elasticsearch commits
		151	$commit = 1 + 0.10 ($slice_index - 1);
		152	}
		153
		154	my %iterator_options;
		155	if ($slice_index) {
		156	_log(1, "Processing slice $slice_index of $slice_count\n");
		157	$iterator_options{slice} = { index => $slice_index, count => $slice_count };
		158	}
123		159
124	my $next;	160	my $next;
125	if ($index_biblios) {	161	if ($index_biblios) {
126	_log(1, "Indexing biblios\n");	162	_log(1, "Indexing biblios\n");
127	if (@biblionumbers) {	163	if (@record_numbers) {
128	$next = sub {	164	$next = sub {
129	my $r = shift @biblionumbers;	165	my $r = shift @record_numbers;
130	return () unless defined $r;	166	return () unless defined $r;
131	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));	167	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132	};	168	};
133	} else {	169	} else {
134	my $records = Koha::BiblioUtils->get_all_biblios_iterator();	170	my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
135	$next = sub {	171	$next = sub {
136	$records->next();	172	$records->next();
137	}	173	}
138	}	174	}
139	do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);	175	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
140	}	176	}
141	if ($index_authorities) {	177	if ($index_authorities) {
142	_log(1, "Indexing authorities\n");	178	_log(1, "Indexing authorities\n");
143	if (@biblionumbers) {	179	if (@record_numbers) {
144	$next = sub {	180	$next = sub {
145	my $r = shift @biblionumbers;	181	my $r = shift @record_numbers;
146	return () unless defined $r;	182	return () unless defined $r;
147	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);	183	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
148	return ($r, $a->record);	184	return ($r, $a->record);
149	};	185	};
150	} else {	186	} else {
151	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	187	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
152	$next = sub {	188	$next = sub {
153	$records->next();	189	$records->next();
154	}	190	}
155	}	191	}
156	do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);	192	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
157	}	193	}
158		194
159	sub do_reindex {	195	if ($slice_index == 1) {
160	my ( $next, $index_name ) = @_;	196	# Main process, wait for children
		197	for (my $proc = 2; $proc <= $processes; $proc++) {
		198	wait();
		199	}
		200	}
		201
		202	=head2 _verify_index_state
		203
		204	_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
		205
		206	Checks the index state and recreates it if requested.
161		207
		208	=cut
		209
		210	sub _verify_index_state {
		211	my ( $index_name, $recreate ) = @_;
		212
		213	_log(1, "Checking state of $index_name index\n");
162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );	214	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163		215
164	if ($delete) {	216	if ($recreate) {
		217	_log(1, "Dropping and recreating $index_name index\n");
165	$indexer->drop_index() if $indexer->index_exists();	218	$indexer->drop_index() if $indexer->index_exists();
166	$indexer->create_index();	219	$indexer->create_index();
167	}	220	}
Lines 174-179 sub do_reindex { Link Here
174	} elsif ($indexer->is_index_status_recreate_required) {	227	} elsif ($indexer->is_index_status_recreate_required) {
175	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;	228	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176	}	229	}
		230	}
		231
		232	=head2 _do_reindex
		233
		234	_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
		235
		236	Does the actual reindexing. $callback is a function that always returns the next record.
		237
		238	=cut
		239
		240	sub _do_reindex {
		241	my ( $next, $index_name ) = @_;
		242
		243	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
177		244
178	my $count = 0;	245	my $count = 0;
179	my $commit_count = $commit;	246	my $commit_count = $commit;
Lines 191-202 sub do_reindex { Link Here
191	push @id_buffer, $id;	258	push @id_buffer, $id;
192	push @commit_buffer, $record;	259	push @commit_buffer, $record;
193	if ( !( --$commit_count ) ) {	260	if ( !( --$commit_count ) ) {
194	_log( 1, "Committing $commit records..." );	261	_log( 1, "Committing $commit records...\n" );
195	$indexer->update_index( \@id_buffer, \@commit_buffer );	262	$indexer->update_index( \@id_buffer, \@commit_buffer );
196	$commit_count = $commit;	263	$commit_count = $commit;
197	@id_buffer = ();	264	@id_buffer = ();
198	@commit_buffer = ();	265	@commit_buffer = ();
199	_log( 1, " done\n" );	266	_log( 1, "Commit complete\n" );
200	}	267	}
201	}	268	}
202		269
Lines 206-226 sub do_reindex { Link Here
206	_log( 1, "Total $count records indexed\n" );	273	_log( 1, "Total $count records indexed\n" );
207	}	274	}
208		275
209	# Checks some basic stuff to ensure that it's sane before we start.	276	=head2 _sanity_check
210	sub sanity_check {	277
		278	_sanity_check();
		279
		280	Checks some basic stuff to ensure that it's sane before we start.
		281
		282	=cut
		283
		284	sub _sanity_check {
211	# Do we have an elasticsearch block defined?	285	# Do we have an elasticsearch block defined?
212	my $conf = C4::Context->config('elasticsearch');	286	my $conf = C4::Context->config('elasticsearch');
213	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );	287	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
214	}	288	}
215		289
216	# Output progress information.	290	=head2 _log
217	#	291
218	# _log($level, $msg);	292	_log($level, "Message\n");
219	#	293
220	# Will output $msg if the verbosity setting is set to $level or more. Will	294	Output progress information.
221	# not include a trailing newline.	295
		296	Will output the message if verbosity level is set to $level or more. Will not
		297	include a trailing newline automatically.
		298
		299	=cut
		300
222	sub _log {	301	sub _log {
223	my ($level, $msg) = @_;	302	my ($level, $msg) = @_;
224		303
225	print $msg if ($verbose >= $level);	304	print "[$$] $msg" if ($verbose >= $level);
226	}	305	}
227	-

Return to bug 21872

Lines 101-120 sub get_from_biblionumber { Link Here

(-)a/Koha/BiblioUtils.pm (-2 / +28 lines)
101		101
102	=head2 get_all_biblios_iterator	102	=head2 get_all_biblios_iterator
103		103
104	my $it = Koha::BiblioUtils->get_all_biblios_iterator();	104	my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
105		105
106	This will provide an iterator object that will, one by one, provide the	106	This will provide an iterator object that will, one by one, provide the
107	Koha::BiblioUtils of each biblio. This will include the item data.	107	Koha::BiblioUtils of each biblio. This will include the item data.
108		108
109	The iterator is a Koha::MetadataIterator object.	109	The iterator is a Koha::MetadataIterator object.
110		110
		111	Possible options are:
		112
		113	=over 4
		114
		115	=item C<slice>
		116
		117	slice may be defined as a hash of two values: index and count. index
		118	is the slice number to process and count is total number of slices.
		119	With this information the iterator returns just the given slice of
		120	records instead of all.
		121
		122	=back
		123
111	=cut	124	=cut
112		125
113	sub get_all_biblios_iterator {	126	sub get_all_biblios_iterator {
		127	my ($self, %options) = @_;
		128
		129	my $search_terms = {};
		130	my ($slice_modulo, $slice_count);
		131	if ($options{slice}) {
		132	$slice_count = $options{slice}->{count};
		133	$slice_modulo = $options{slice}->{index};
		134	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		135
		136	$search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo];
		137	}
		138
114	my $database = Koha::Database->new();	139	my $database = Koha::Database->new();
115	my $schema = $database->schema();	140	my $schema = $database->schema();
116	my $rs =	141	my $rs =
117	$schema->resultset('Biblio')->search( {},	142	$schema->resultset('Biblio')->search(
		143	$search_terms,
118	{ columns => [qw/ biblionumber /] } );	144	{ columns => [qw/ biblionumber /] } );
119	my $next_func = sub {	145	my $next_func = sub {
120	# Warn and skip bad records, otherwise we break the loop	146	# Warn and skip bad records, otherwise we break the loop

Lines 150-169 sub authorized_heading { Link Here

(-)a/Koha/MetadataRecord/Authority.pm (-2 / +36 lines)
150		150
151	=head2 get_all_authorities_iterator	151	=head2 get_all_authorities_iterator
152		152
153	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator();	153	my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
154		154
155	This will provide an iterator object that will, one by one, provide the	155	This will provide an iterator object that will, one by one, provide the
156	Koha::MetadataRecord::Authority of each authority.	156	Koha::MetadataRecord::Authority of each authority.
157		157
158	The iterator is a Koha::MetadataIterator object.	158	The iterator is a Koha::MetadataIterator object.
159		159
		160	Possible options are:
		161
		162	=over 4
		163
		164	=item C<slice>
		165
		166	slice may be defined as a hash of two values: index and count. index
		167	is the slice number to process and count is total number of slices.
		168	With this information the iterator returns just the given slice of
		169	records instead of all.
		170
		171	=back
		172
160	=cut	173	=cut
161		174
162	sub get_all_authorities_iterator {	175	sub get_all_authorities_iterator {
		176	my ($self, %options) = @_;
		177
		178	my $search_terms = {
		179	marcxml => { '!=', undef }
		180	};
		181	my ($slice_modulo, $slice_count);
		182	if ($options{slice}) {
		183	$slice_count = $options{slice}->{count};
		184	$slice_modulo = $options{slice}->{index};
		185	$slice_modulo = 0 if ($slice_modulo == $slice_count);
		186
		187	$search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo];
		188	$search_terms = {
		189	'-and' => [
		190	$search_terms,
		191	\[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo]
		192	]
		193	};
		194	}
		195
163	my $database = Koha::Database->new();	196	my $database = Koha::Database->new();
164	my $schema = $database->schema();	197	my $schema = $database->schema();
165	my $rs =	198	my $rs =
166	$schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },	199	$schema->resultset('AuthHeader')->search(
		200	$search_terms,
167	{ columns => [qw/ authid authtypecode marcxml /] } );	201	{ columns => [qw/ authid authtypecode marcxml /] } );
168	my $next_func = sub {	202	my $next_func = sub {
169	my $row = $rs->next();	203	my $row = $rs->next();