Attachment #82586 for bug #21872

View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All




#!/usr/bin/perl

=head1 NAME

record_batches.pl - Outputs biblio record batches as ranges of biblionumbers.

=head1 SYNOPSIS

record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER]

=head1 OPTIONS

=over

=item B<-h|--help>

Print a brief help message.

=item B<--startbiblionumber>

 --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER

=item B<--batchsize>

=back

=cut

use Modern::Perl;
use C4::Context;
use Getopt::Long;

    $start_biblionumber = $min_biblionumber;
}

# TODO: Could be options
my $range_separator = " ";
my $batch_separator = "\n";
sub print_range {
    print join($range_separator, @_) . $batch_separator;
}

while (1) {
    my $offset = $batch_size - 1;
    my $query =

            my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };
            if($max_biblionumber) {
                if($max_biblionumber >= $start_biblionumber) {
                    print_range($start_biblionumber, $max_biblionumber);
                    print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n";
                }
            }
        }
        last;
    }
    else {
        print_range($start_biblionumber, $end_biblionumber);
    }
    if ($next_biblionumber) {
        $start_biblionumber = $next_biblionumber;

        last;
    }
}

=head1 NAME

record batches - This script outputs ranges of biblionumbers

=head1 SYNOPSIS

record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER]

=head1 OPTIONS

=over

=item B<-h|--help>

Print a brief help message.

=item B<--startbiblionumber>

 --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER

=item B<--batchsize>




#!/usr/bin/perl

=head1 NAME

parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel.

=head1 SYNOPSIS

B<parallel_rebuild_biblios.pl>
[B<-h|--help>]
[B<-v|--verbose>]
[B<-j|--jobs>=C<jobs>]
[B<-b|--batchsize>=C<size>]
[B<-c|--commit>=C<size>]
[B<--startbiblionumber>=C<biblionumber>]
[B<-d|--delete>]

=head1 OPTIONS

=over

=item B<-h|--help>

Print a brief help message.

=item B<-v|--verbose>

Passed to rebuild_elastic_search.pl. Increase verbosity.

=item B<-j|--jobs>=C<N>

Run up to N jobs in parallel.  0 means as many as possible. Default is to run
one job per CPU core. This is passed directly to parrallel as B<-j>C<N>. Se C<man parrallel>
for more information about variants of C<N>.

=item B<-b|--batchsize>=C<size>

Specify batch size for batches of biblios passed to rebuild_elastic_search.pl.

=item B<-c|--commit>=C<size>

Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch.

=item B<--startbiblionumber>=C<biblionumber>

Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed.

=item B<-d|--delete>

Delete the index and recreate it before indexing.

=back

=cut

use Modern::Perl;
use Getopt::Long;
use Pod::Usage;
use Koha::SearchEngine::Elasticsearch::Indexer;
use C4::Context;


#TODO: Is the the correct way of getting koha root?
my $koha_root = C4::Context->config('intranetdir');

my (
    $help,
    $man,
    $verbose,
    $jobs,
    $batch_size,
    $commit_size,
    $start_biblionumber,
    $delete
);

# Default option values
$batch_size = 30000;
GetOptions(
    'h|help' => \$help,
    'man' => \$man,
    'v|verbose' => \$verbose,
    'j|jobs=i' => \$jobs,
    'b|batchsize=i' => \$batch_size,
    'c|commit=i' => \$commit_size,
    'startbiblionumber=i' => \$start_biblionumber,
    'd|delete' => \$delete
);

pod2usage(1) if $help;
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;

if ($delete) {
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({
        index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
    });
    $indexer->drop_index();
    $indexer->create_index();
}

# record_batches.pl options
my @record_batches_opts;
if ($start_biblionumber) {
    push @record_batches_opts, "--startbiblionumber=$start_biblionumber";
}
push @record_batches_opts, "--batchsize=$batch_size";
my $record_batches_opts = join(' ', @record_batches_opts);

# rebuild_elastic_search.pl options
my @rebuild_elastic_opts = ('-b');
if ($verbose) {
    push @rebuild_elastic_opts, "-v";
}
if ($commit_size) {
    push @rebuild_elastic_opts, "--commit=$commit_size";
}
my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts);

# parallel options
my @parallel_opts;
if ($jobs) {
    push @parallel_opts, "-j$jobs";
}
my $parallel_opts = join(' ', @parallel_opts);

my $command = join('', (
    "$koha_root/misc/record_batches.pl $record_batches_opts | ",
    "parallel $parallel_opts --colsep ' ' ",
    "$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}"
));

exec $command;




repeated. This also applies to authorities via authid, so if you're using it,
you probably only want to do one or the other at a time.

=item B<-sbn|--startbiblionumber>

Only index biblios with biblionumber greater or equal than supplied
biblionumber.

=item B<-ebn|--endbiblionumber>

Only index biblios with biblionumber less or equal to supplied biblionumber.


    'a|authorities' => \$index_authorities,
    'b|biblios' => \$index_biblios,
    'bn|bnumber=i' => \@biblionumbers,
    'sbn|startbiblionumber=i' => \$start_biblionumber,
    'ebn|endbiblionumber=i' => \$end_biblionumber,
    'v|verbose+'       => \$verbose,
    'h|help'           => \$help,
    'man'              => \$man,
- 

Return to bug 21872

Lines 1-5 Link Here

(-)a/misc/record_batches.pl (-25 / +35 lines)
1	#!/usr/bin/perl	1	#!/usr/bin/perl
2		2
		3	=head1 NAME
		4
		5	record_batches.pl - Outputs biblio record batches as ranges of biblionumbers.
		6
		7	=head1 SYNOPSIS
		8
		9	record_batches.pl [-h\|--help] [--startbiblionumber=BIBLIONUMBER]
		10
		11	=head1 OPTIONS
		12
		13	=over
		14
		15	=item B<-h\|--help>
		16
		17	Print a brief help message.
		18
		19	=item B<--startbiblionumber>
		20
		21	--startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER
		22
		23	=item B<--batchsize>
		24
		25	=back
		26
		27	=cut
		28
3	use Modern::Perl;	29	use Modern::Perl;
4	use C4::Context;	30	use C4::Context;
5	use Getopt::Long;	31	use Getopt::Long;
Lines 25-30 if (!$start_biblionumber) { Link Here
25	$start_biblionumber = $min_biblionumber;	51	$start_biblionumber = $min_biblionumber;
26	}	52	}
27		53
		54	# TODO: Could be options
		55	my $range_separator = " ";
		56	my $batch_separator = "\n";
		57	sub print_range {
		58	print join($range_separator, @_) . $batch_separator;
		59	}
		60
28	while (1) {	61	while (1) {
29	my $offset = $batch_size - 1;	62	my $offset = $batch_size - 1;
30	my $query =	63	my $query =
Lines 35-49 while (1) { Link Here
35	my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };	68	my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };
36	if($max_biblionumber) {	69	if($max_biblionumber) {
37	if($max_biblionumber >= $start_biblionumber) {	70	if($max_biblionumber >= $start_biblionumber) {
38	# @TODO: Option for argument and range separator?	71	print_range($start_biblionumber, $max_biblionumber);
39	print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n";
40	}	72	}
41	}	73	}
42	}	74	}
43	last;	75	last;
44	}	76	}
45	else {	77	else {
46	print join(' ', ($start_biblionumber, $end_biblionumber)) . "\n";	78	print_range($start_biblionumber, $end_biblionumber);
47	}	79	}
48	if ($next_biblionumber) {	80	if ($next_biblionumber) {
49	$start_biblionumber = $next_biblionumber;	81	$start_biblionumber = $next_biblionumber;
Lines 52-76 while (1) { Link Here
52	last;	84	last;
53	}	85	}
54	}	86	}
55
56	=head1 NAME
57
58	record batches - This script outputs ranges of biblionumbers
59
60	=head1 SYNOPSIS
61
62	record_batches.pl [-h\|--help] [--startbiblionumber=BIBLIONUMBER]
63
64	=head1 OPTIONS
65
66	=over
67
68	=item B<-h\|--help>
69
70	Print a brief help message.
71
72	=item B<--startbiblionumber>
73
74	--startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER
75
76	=item B<--batchsize>

Line 0 Link Here

(-)a/misc/search_tools/parallel_rebuild_biblios.pl (+132 lines)
		1	#!/usr/bin/perl
		2
		3	=head1 NAME
		4
		5	parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel.
		6
		7	=head1 SYNOPSIS
		8
		9	B<parallel_rebuild_biblios.pl>
		10	[B<-h\|--help>]
		11	[B<-v\|--verbose>]
		12	[B<-j\|--jobs>=C<jobs>]
		13	[B<-b\|--batchsize>=C<size>]
		14	[B<-c\|--commit>=C<size>]
		15	[B<--startbiblionumber>=C<biblionumber>]
		16	[B<-d\|--delete>]
		17
		18	=head1 OPTIONS
		19
		20	=over
		21
		22	=item B<-h\|--help>
		23
		24	Print a brief help message.
		25
		26	=item B<-v\|--verbose>
		27
		28	Passed to rebuild_elastic_search.pl. Increase verbosity.
		29
		30	=item B<-j\|--jobs>=C<N>
		31
		32	Run up to N jobs in parallel. 0 means as many as possible. Default is to run
		33	one job per CPU core. This is passed directly to parrallel as B<-j>C<N>. Se C<man parrallel>
		34	for more information about variants of C<N>.
		35
		36	=item B<-b\|--batchsize>=C<size>
		37
		38	Specify batch size for batches of biblios passed to rebuild_elastic_search.pl.
		39
		40	=item B<-c\|--commit>=C<size>
		41
		42	Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch.
		43
		44	=item B<--startbiblionumber>=C<biblionumber>
		45
		46	Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed.
		47
		48	=item B<-d\|--delete>
		49
		50	Delete the index and recreate it before indexing.
		51
		52	=back
		53
		54	=cut
		55
		56	use Modern::Perl;
		57	use Getopt::Long;
		58	use Pod::Usage;
		59	use Koha::SearchEngine::Elasticsearch::Indexer;
		60	use C4::Context;
		61
		62
		63	#TODO: Is the the correct way of getting koha root?
		64	my $koha_root = C4::Context->config('intranetdir');
65
66	my (
67	$help,
68	$man,
69	$verbose,
70	$jobs,
71	$batch_size,
72	$commit_size,
73	$start_biblionumber,
74	$delete
75	);
76
77	# Default option values
78	$batch_size = 30000;
79	GetOptions(
80	'h\|help' => \$help,
81	'man' => \$man,
82	'v\|verbose' => \$verbose,
83	'j\|jobs=i' => \$jobs,
84	'b\|batchsize=i' => \$batch_size,
85	'c\|commit=i' => \$commit_size,
86	'startbiblionumber=i' => \$start_biblionumber,
87	'd\|delete' => \$delete
88	);
89
90	pod2usage(1) if $help;
91	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
92
93	if ($delete) {
94	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({
95	index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
96	});
97	$indexer->drop_index();
98	$indexer->create_index();
99	}
100
101	# record_batches.pl options
102	my @record_batches_opts;
103	if ($start_biblionumber) {
104	push @record_batches_opts, "--startbiblionumber=$start_biblionumber";
105	}
106	push @record_batches_opts, "--batchsize=$batch_size";
107	my $record_batches_opts = join(' ', @record_batches_opts);
108
109	# rebuild_elastic_search.pl options
110	my @rebuild_elastic_opts = ('-b');
111	if ($verbose) {
112	push @rebuild_elastic_opts, "-v";
113	}
114	if ($commit_size) {
115	push @rebuild_elastic_opts, "--commit=$commit_size";
116	}
117	my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts);
118
119	# parallel options
120	my @parallel_opts;
121	if ($jobs) {
122	push @parallel_opts, "-j$jobs";
123	}
124	my $parallel_opts = join(' ', @parallel_opts);
125
126	my $command = join('', (
127	"$koha_root/misc/record_batches.pl $record_batches_opts \| ",
128	"parallel $parallel_opts --colsep ' ' ",
129	"$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}"
130	));
131
132	exec $command;

Lines 64-75 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-5 / +4 lines)
64	repeated. This also applies to authorities via authid, so if you're using it,	64	repeated. This also applies to authorities via authid, so if you're using it,
65	you probably only want to do one or the other at a time.	65	you probably only want to do one or the other at a time.
66		66
67	=item B<-sbn\|--start-bnumber>	67	=item B<-sbn\|--startbiblionumber>
68		68
69	Only index biblios with biblionumber greater or equal than supplied	69	Only index biblios with biblionumber greater or equal than supplied
70	biblionumber.	70	biblionumber.
71		71
72	=item B<-ebn\|--end-bnumber>	72	=item B<-ebn\|--endbiblionumber>
73		73
74	Only index biblios with biblionumber less or equal to supplied biblionumber.	74	Only index biblios with biblionumber less or equal to supplied biblionumber.
75		75
Lines 122-129 GetOptions( Link Here
122	'a\|authorities' => \$index_authorities,	122	'a\|authorities' => \$index_authorities,
123	'b\|biblios' => \$index_biblios,	123	'b\|biblios' => \$index_biblios,
124	'bn\|bnumber=i' => \@biblionumbers,	124	'bn\|bnumber=i' => \@biblionumbers,
125	'sbn\|start-bnumber=i' => \$start_biblionumber,	125	'sbn\|startbiblionumber=i' => \$start_biblionumber,
126	'ebn\|end-bnumber=i' => \$end_biblionumber,	126	'ebn\|endbiblionumber=i' => \$end_biblionumber,
127	'v\|verbose+' => \$verbose,	127	'v\|verbose+' => \$verbose,
128	'h\|help' => \$help,	128	'h\|help' => \$help,
129	'man' => \$man,	129	'man' => \$man,
130	-