Lines 17-34
Link Here
|
17 |
|
17 |
|
18 |
use Modern::Perl; |
18 |
use Modern::Perl; |
19 |
|
19 |
|
20 |
use Koha::Script; |
20 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
21 |
use C4::Context; |
21 |
use File::Temp qw( tempdir ); |
|
|
22 |
use File::Path qw( mkpath rmtree ); |
22 |
use Getopt::Long qw( GetOptions ); |
23 |
use Getopt::Long qw( GetOptions ); |
23 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
24 |
use Parallel::ForkManager; |
24 |
use File::Temp qw( tempdir ); |
25 |
use POSIX qw/ceil/; |
25 |
use File::Path qw( mkpath rmtree ); |
26 |
use XML::LibXML; |
26 |
use C4::Biblio qw( GetXmlBiblio ); |
27 |
|
|
|
28 |
use C4::Context; |
27 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
29 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
28 |
use C4::Items qw( Item2Marc ); |
30 |
use C4::Biblio qw( GetXmlBiblio ); |
29 |
use Koha::RecordProcessor; |
31 |
use C4::Items qw( Item2Marc ); |
|
|
32 |
use Koha::Authorities; |
33 |
use Koha::Biblios; |
30 |
use Koha::Caches; |
34 |
use Koha::Caches; |
31 |
use XML::LibXML; |
35 |
use Koha::RecordProcessor; |
|
|
36 |
use Koha::Script; |
32 |
|
37 |
|
33 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
38 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
34 |
|
39 |
|
Lines 63-69
my $run_as_root;
Link Here
|
63 |
my $run_user = (getpwuid($<))[0]; |
68 |
my $run_user = (getpwuid($<))[0]; |
64 |
my $wait_for_lock = 0; |
69 |
my $wait_for_lock = 0; |
65 |
my $use_flock; |
70 |
my $use_flock; |
66 |
my $table = 'biblioitems'; |
71 |
my $table = 'biblioitems'; |
|
|
72 |
my $forks = 0; |
73 |
my $chunk_size = 100000; |
67 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
74 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
68 |
|
75 |
|
69 |
my $verbose_logging = 0; |
76 |
my $verbose_logging = 0; |
Lines 92-97
my $result = GetOptions(
Link Here
|
92 |
'run-as-root' => \$run_as_root, |
99 |
'run-as-root' => \$run_as_root, |
93 |
'wait-for-lock' => \$wait_for_lock, |
100 |
'wait-for-lock' => \$wait_for_lock, |
94 |
't|table:s' => \$table, |
101 |
't|table:s' => \$table, |
|
|
102 |
'forks:i' => \$forks, |
95 |
); |
103 |
); |
96 |
|
104 |
|
97 |
if (not $result or $want_help) { |
105 |
if (not $result or $want_help) { |
Lines 388-395
sub index_records {
Link Here
|
388 |
mark_zebraqueue_batch_done($entries); |
396 |
mark_zebraqueue_batch_done($entries); |
389 |
|
397 |
|
390 |
} else { |
398 |
} else { |
391 |
my $sth = select_all_records($record_type); |
399 |
$num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize ); |
392 |
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize); |
|
|
393 |
unless ($do_not_clear_zebraqueue) { |
400 |
unless ($do_not_clear_zebraqueue) { |
394 |
mark_all_zebraqueue_done($record_type); |
401 |
mark_all_zebraqueue_done($record_type); |
395 |
} |
402 |
} |
Lines 474-480
sub select_all_authorities {
Link Here
|
474 |
$strsth.=qq{ WHERE $where } if ($where); |
481 |
$strsth.=qq{ WHERE $where } if ($where); |
475 |
$strsth.=qq{ LIMIT $length } if ($length && !$offset); |
482 |
$strsth.=qq{ LIMIT $length } if ($length && !$offset); |
476 |
$strsth.=qq{ LIMIT $offset,$length } if ($length && $offset); |
483 |
$strsth.=qq{ LIMIT $offset,$length } if ($length && $offset); |
477 |
my $sth = $dbh->prepare($strsth); |
484 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
|
|
485 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
478 |
$sth->execute(); |
486 |
$sth->execute(); |
479 |
return $sth; |
487 |
return $sth; |
480 |
} |
488 |
} |
Lines 486-501
sub select_all_biblios {
Link Here
|
486 |
$strsth.=qq{ WHERE $where } if ($where); |
494 |
$strsth.=qq{ WHERE $where } if ($where); |
487 |
$strsth.=qq{ LIMIT $length } if ($length && !$offset); |
495 |
$strsth.=qq{ LIMIT $length } if ($length && !$offset); |
488 |
$strsth.=qq{ LIMIT $offset,$length } if ($offset); |
496 |
$strsth.=qq{ LIMIT $offset,$length } if ($offset); |
489 |
my $sth = $dbh->prepare($strsth); |
497 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
|
|
498 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
490 |
$sth->execute(); |
499 |
$sth->execute(); |
491 |
return $sth; |
500 |
return $sth; |
492 |
} |
501 |
} |
493 |
|
502 |
|
|
|
503 |
sub export_marc_records { |
504 |
my ($record_type, $directory, $nosanitize) = @_; |
505 |
my $pm = Parallel::ForkManager->new($forks); |
506 |
my @original_params = ( $offset, $length ); |
507 |
$offset ||= 0; |
508 |
$length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count ); |
509 |
my $chunk_size = $forks ? ceil( $length / $forks ) : $length; |
510 |
my ( $seq, $num_records_exported ) = ( undef, 0 ); |
511 |
while ( $chunk_size > 0 ) { |
512 |
# If you use forks, ->start forks after getting process slot |
513 |
unless ( $pm->start ) { |
514 |
# Child code (or parent when forks parameter is absent) |
515 |
$length = $chunk_size; |
516 |
my $sth = select_all_records($record_type); |
517 |
export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq ); |
518 |
$pm->finish; # exit for child only |
519 |
} |
520 |
$offset += $chunk_size; |
521 |
$num_records_exported += $chunk_size; |
522 |
$seq++; |
523 |
$chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length; |
524 |
} |
525 |
$pm->wait_all_children; |
526 |
( $offset, $length ) = @original_params; # restore for a potential second run (with -a -b) |
527 |
return $num_records_exported; |
528 |
} |
529 |
|
494 |
sub export_marc_records_from_sth { |
530 |
sub export_marc_records_from_sth { |
495 |
my ($record_type, $sth, $directory, $nosanitize) = @_; |
531 |
my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_; |
|
|
532 |
$seq ||= q{}; |
496 |
|
533 |
|
497 |
my $num_exported = 0; |
534 |
my $num_exported = 0; |
498 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
535 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!; |
499 |
|
536 |
|
500 |
print {$fh} $marcxml_open; |
537 |
print {$fh} $marcxml_open; |
501 |
|
538 |
|
502 |
- |
|
|