View | Details | Raw Unified | Return to bug 21872
Collapse All | Expand All

(-)a/misc/record_batches.pl (-25 / +35 lines)
Lines 1-5 Link Here
1
#!/usr/bin/perl
1
#!/usr/bin/perl
2
2
3
=head1 NAME
4
5
record_batches.pl - Outputs biblio record batches as ranges of biblionumbers.
6
7
=head1 SYNOPSIS
8
9
record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER]
10
11
=head1 OPTIONS
12
13
=over
14
15
=item B<-h|--help>
16
17
Print a brief help message.
18
19
=item B<--startbiblionumber>
20
21
 --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER
22
23
=item B<--batchsize>
24
25
=back
26
27
=cut
28
3
use Modern::Perl;
29
use Modern::Perl;
4
use C4::Context;
30
use C4::Context;
5
use Getopt::Long;
31
use Getopt::Long;
Lines 25-30 if (!$start_biblionumber) { Link Here
25
    $start_biblionumber = $min_biblionumber;
51
    $start_biblionumber = $min_biblionumber;
26
}
52
}
27
53
54
# TODO: Could be options
55
my $range_separator = " ";
56
my $batch_separator = "\n";
57
sub print_range {
58
    print join($range_separator, @_) . $batch_separator;
59
}
60
28
while (1) {
61
while (1) {
29
    my $offset = $batch_size - 1;
62
    my $offset = $batch_size - 1;
30
    my $query =
63
    my $query =
Lines 35-49 while (1) { Link Here
35
            my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };
68
            my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) };
36
            if($max_biblionumber) {
69
            if($max_biblionumber) {
37
                if($max_biblionumber >= $start_biblionumber) {
70
                if($max_biblionumber >= $start_biblionumber) {
38
                    # @TODO: Option for argument and range separator?
71
                    print_range($start_biblionumber, $max_biblionumber);
39
                    print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n";
40
                }
72
                }
41
            }
73
            }
42
        }
74
        }
43
        last;
75
        last;
44
    }
76
    }
45
    else {
77
    else {
46
        print join(' ', ($start_biblionumber, $end_biblionumber)) . "\n";
78
        print_range($start_biblionumber, $end_biblionumber);
47
    }
79
    }
48
    if ($next_biblionumber) {
80
    if ($next_biblionumber) {
49
        $start_biblionumber = $next_biblionumber;
81
        $start_biblionumber = $next_biblionumber;
Lines 52-76 while (1) { Link Here
52
        last;
84
        last;
53
    }
85
    }
54
}
86
}
55
56
=head1 NAME
57
58
record batches - This script outputs ranges of biblionumbers
59
60
=head1 SYNOPSIS
61
62
record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER]
63
64
=head1 OPTIONS
65
66
=over
67
68
=item B<-h|--help>
69
70
Print a brief help message.
71
72
=item B<--startbiblionumber>
73
74
 --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER
75
76
=item B<--batchsize>
(-)a/misc/search_tools/parallel_rebuild_biblios.pl (+132 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
=head1 NAME
4
5
parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel.
6
7
=head1 SYNOPSIS
8
9
B<parallel_rebuild_biblios.pl>
10
[B<-h|--help>]
11
[B<-v|--verbose>]
12
[B<-j|--jobs>=C<jobs>]
13
[B<-b|--batchsize>=C<size>]
14
[B<-c|--commit>=C<size>]
15
[B<--startbiblionumber>=C<biblionumber>]
16
[B<-d|--delete>]
17
18
=head1 OPTIONS
19
20
=over
21
22
=item B<-h|--help>
23
24
Print a brief help message.
25
26
=item B<-v|--verbose>
27
28
Passed to rebuild_elastic_search.pl. Increase verbosity.
29
30
=item B<-j|--jobs>=C<N>
31
32
Run up to N jobs in parallel.  0 means as many as possible. Default is to run
33
one job per CPU core. This is passed directly to parrallel as B<-j>C<N>. Se C<man parrallel>
34
for more information about variants of C<N>.
35
36
=item B<-b|--batchsize>=C<size>
37
38
Specify batch size for batches of biblios passed to rebuild_elastic_search.pl.
39
40
=item B<-c|--commit>=C<size>
41
42
Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch.
43
44
=item B<--startbiblionumber>=C<biblionumber>
45
46
Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed.
47
48
=item B<-d|--delete>
49
50
Delete the index and recreate it before indexing.
51
52
=back
53
54
=cut
55
56
use Modern::Perl;
57
use Getopt::Long;
58
use Pod::Usage;
59
use Koha::SearchEngine::Elasticsearch::Indexer;
60
use C4::Context;
61
62
63
#TODO: Is the the correct way of getting koha root?
64
my $koha_root = C4::Context->config('intranetdir');
65
66
my (
67
    $help,
68
    $man,
69
    $verbose,
70
    $jobs,
71
    $batch_size,
72
    $commit_size,
73
    $start_biblionumber,
74
    $delete
75
);
76
77
# Default option values
78
$batch_size = 30000;
79
GetOptions(
80
    'h|help' => \$help,
81
    'man' => \$man,
82
    'v|verbose' => \$verbose,
83
    'j|jobs=i' => \$jobs,
84
    'b|batchsize=i' => \$batch_size,
85
    'c|commit=i' => \$commit_size,
86
    'startbiblionumber=i' => \$start_biblionumber,
87
    'd|delete' => \$delete
88
);
89
90
pod2usage(1) if $help;
91
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
92
93
if ($delete) {
94
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({
95
        index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
96
    });
97
    $indexer->drop_index();
98
    $indexer->create_index();
99
}
100
101
# record_batches.pl options
102
my @record_batches_opts;
103
if ($start_biblionumber) {
104
    push @record_batches_opts, "--startbiblionumber=$start_biblionumber";
105
}
106
push @record_batches_opts, "--batchsize=$batch_size";
107
my $record_batches_opts = join(' ', @record_batches_opts);
108
109
# rebuild_elastic_search.pl options
110
my @rebuild_elastic_opts = ('-b');
111
if ($verbose) {
112
    push @rebuild_elastic_opts, "-v";
113
}
114
if ($commit_size) {
115
    push @rebuild_elastic_opts, "--commit=$commit_size";
116
}
117
my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts);
118
119
# parallel options
120
my @parallel_opts;
121
if ($jobs) {
122
    push @parallel_opts, "-j$jobs";
123
}
124
my $parallel_opts = join(' ', @parallel_opts);
125
126
my $command = join('', (
127
    "$koha_root/misc/record_batches.pl $record_batches_opts | ",
128
    "parallel $parallel_opts --colsep ' ' ",
129
    "$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}"
130
));
131
132
exec $command;
(-)a/misc/search_tools/rebuild_elastic_search.pl (-5 / +4 lines)
Lines 64-75 Only index the supplied biblionumber, mostly for testing purposes. May be Link Here
64
repeated. This also applies to authorities via authid, so if you're using it,
64
repeated. This also applies to authorities via authid, so if you're using it,
65
you probably only want to do one or the other at a time.
65
you probably only want to do one or the other at a time.
66
66
67
=item B<-sbn|--start-bnumber>
67
=item B<-sbn|--startbiblionumber>
68
68
69
Only index biblios with biblionumber greater or equal than supplied
69
Only index biblios with biblionumber greater or equal than supplied
70
biblionumber.
70
biblionumber.
71
71
72
=item B<-ebn|--end-bnumber>
72
=item B<-ebn|--endbiblionumber>
73
73
74
Only index biblios with biblionumber less or equal to supplied biblionumber.
74
Only index biblios with biblionumber less or equal to supplied biblionumber.
75
75
Lines 122-129 GetOptions( Link Here
122
    'a|authorities' => \$index_authorities,
122
    'a|authorities' => \$index_authorities,
123
    'b|biblios' => \$index_biblios,
123
    'b|biblios' => \$index_biblios,
124
    'bn|bnumber=i' => \@biblionumbers,
124
    'bn|bnumber=i' => \@biblionumbers,
125
    'sbn|start-bnumber=i' => \$start_biblionumber,
125
    'sbn|startbiblionumber=i' => \$start_biblionumber,
126
    'ebn|end-bnumber=i' => \$end_biblionumber,
126
    'ebn|endbiblionumber=i' => \$end_biblionumber,
127
    'v|verbose+'       => \$verbose,
127
    'v|verbose+'       => \$verbose,
128
    'h|help'           => \$help,
128
    'h|help'           => \$help,
129
    'man'              => \$man,
129
    'man'              => \$man,
130
- 

Return to bug 21872