Lines 17-34
Link Here
|
17 |
|
17 |
|
18 |
use Modern::Perl; |
18 |
use Modern::Perl; |
19 |
|
19 |
|
20 |
use Koha::Script; |
20 |
use Getopt::Long qw( GetOptions ); |
|
|
21 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
22 |
use File::Temp qw( tempdir ); |
23 |
use File::Path qw( mkpath rmtree ); |
24 |
use Parallel::ForkManager; |
25 |
use POSIX qw/ceil/; |
26 |
use XML::LibXML; |
27 |
|
21 |
use C4::Context; |
28 |
use C4::Context; |
22 |
use Getopt::Long qw( GetOptions ); |
|
|
23 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
24 |
use File::Temp qw( tempdir ); |
25 |
use File::Path qw( mkpath rmtree ); |
26 |
use C4::Biblio qw( GetXmlBiblio ); |
27 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
29 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
|
|
30 |
use C4::Biblio qw( GetXmlBiblio ); |
28 |
use C4::Items qw( Item2Marc ); |
31 |
use C4::Items qw( Item2Marc ); |
29 |
use Koha::RecordProcessor; |
32 |
use Koha::Authorities; |
|
|
33 |
use Koha::Biblios; |
30 |
use Koha::Caches; |
34 |
use Koha::Caches; |
31 |
use XML::LibXML; |
35 |
use Koha::RecordProcessor; |
|
|
36 |
use Koha::Script; |
32 |
|
37 |
|
33 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
38 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
34 |
|
39 |
|
Lines 65-70
my $run_user = ( getpwuid($<) )[0];
Link Here
|
65 |
my $wait_for_lock = 0; |
70 |
my $wait_for_lock = 0; |
66 |
my $use_flock; |
71 |
my $use_flock; |
67 |
my $table = 'biblioitems'; |
72 |
my $table = 'biblioitems'; |
|
|
73 |
my $forks = 0; |
74 |
my $chunk_size = 100000; |
68 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
75 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
69 |
|
76 |
|
70 |
my $verbose_logging = 0; |
77 |
my $verbose_logging = 0; |
Lines 93-98
my $result = GetOptions(
Link Here
|
93 |
'run-as-root' => \$run_as_root, |
100 |
'run-as-root' => \$run_as_root, |
94 |
'wait-for-lock' => \$wait_for_lock, |
101 |
'wait-for-lock' => \$wait_for_lock, |
95 |
't|table:s' => \$table, |
102 |
't|table:s' => \$table, |
|
|
103 |
'forks:i' => \$forks, |
96 |
); |
104 |
); |
97 |
|
105 |
|
98 |
if ( not $result or $want_help ) { |
106 |
if ( not $result or $want_help ) { |
Lines 409-417
sub index_records {
Link Here
|
409 |
mark_zebraqueue_batch_done($entries); |
417 |
mark_zebraqueue_batch_done($entries); |
410 |
|
418 |
|
411 |
} else { |
419 |
} else { |
412 |
my $sth = select_all_records($record_type); |
420 |
$num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize ); |
413 |
$num_records_exported = |
|
|
414 |
export_marc_records_from_sth( $record_type, $sth, "$directory/$record_type", $nosanitize ); |
415 |
unless ($do_not_clear_zebraqueue) { |
421 |
unless ($do_not_clear_zebraqueue) { |
416 |
mark_all_zebraqueue_done($record_type); |
422 |
mark_all_zebraqueue_done($record_type); |
417 |
} |
423 |
} |
Lines 505-511
sub select_all_authorities {
Link Here
|
505 |
$strsth .= qq{ WHERE $where } if ($where); |
511 |
$strsth .= qq{ WHERE $where } if ($where); |
506 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
512 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
507 |
$strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); |
513 |
$strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); |
508 |
my $sth = $dbh->prepare($strsth); |
514 |
|
|
|
515 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
516 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
509 |
$sth->execute(); |
517 |
$sth->execute(); |
510 |
return $sth; |
518 |
return $sth; |
511 |
} |
519 |
} |
Lines 517-532
sub select_all_biblios {
Link Here
|
517 |
$strsth .= qq{ WHERE $where } if ($where); |
525 |
$strsth .= qq{ WHERE $where } if ($where); |
518 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
526 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
519 |
$strsth .= qq{ LIMIT $offset,$length } if ($offset); |
527 |
$strsth .= qq{ LIMIT $offset,$length } if ($offset); |
520 |
my $sth = $dbh->prepare($strsth); |
528 |
|
|
|
529 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
530 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
521 |
$sth->execute(); |
531 |
$sth->execute(); |
522 |
return $sth; |
532 |
return $sth; |
523 |
} |
533 |
} |
524 |
|
534 |
|
|
|
535 |
sub export_marc_records { |
536 |
my ( $record_type, $directory, $nosanitize ) = @_; |
537 |
my $pm = Parallel::ForkManager->new($forks); |
538 |
my @original_params = ( $offset, $length ); |
539 |
$offset ||= 0; |
540 |
$length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count ); |
541 |
my $chunk_size = $forks ? ceil( $length / $forks ) : $length; |
542 |
my ( $seq, $num_records_exported ) = ( undef, 0 ); |
543 |
while ( $chunk_size > 0 ) { |
544 |
|
545 |
# If you use forks, ->start forks after getting process slot |
546 |
unless ( $pm->start ) { |
547 |
|
548 |
# Child code (or parent when forks parameter is absent) |
549 |
$length = $chunk_size; |
550 |
my $sth = select_all_records($record_type); |
551 |
export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq ); |
552 |
$pm->finish; # exit for child only |
553 |
} |
554 |
$offset += $chunk_size; |
555 |
$num_records_exported += $chunk_size; |
556 |
$seq++; |
557 |
$chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length; |
558 |
} |
559 |
$pm->wait_all_children; |
560 |
( $offset, $length ) = @original_params; # restore for a potential second run (with -a -b) |
561 |
return $num_records_exported; |
562 |
} |
563 |
|
525 |
sub export_marc_records_from_sth { |
564 |
sub export_marc_records_from_sth { |
526 |
my ( $record_type, $sth, $directory, $nosanitize ) = @_; |
565 |
my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_; |
|
|
566 |
$seq ||= q{}; |
527 |
|
567 |
|
528 |
my $num_exported = 0; |
568 |
my $num_exported = 0; |
529 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
569 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!; |
530 |
|
570 |
|
531 |
print {$fh} $marcxml_open; |
571 |
print {$fh} $marcxml_open; |
532 |
|
572 |
|
533 |
- |
|
|