|
Lines 17-34
Link Here
|
| 17 |
|
17 |
|
| 18 |
use Modern::Perl; |
18 |
use Modern::Perl; |
| 19 |
|
19 |
|
| 20 |
use Koha::Script; |
20 |
use Getopt::Long qw( GetOptions ); |
|
|
21 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
| 22 |
use File::Temp qw( tempdir ); |
| 23 |
use File::Path qw( mkpath rmtree ); |
| 24 |
use Parallel::ForkManager; |
| 25 |
use POSIX qw/ceil/; |
| 26 |
use XML::LibXML; |
| 27 |
|
| 21 |
use C4::Context; |
28 |
use C4::Context; |
| 22 |
use Getopt::Long qw( GetOptions ); |
|
|
| 23 |
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN ); |
| 24 |
use File::Temp qw( tempdir ); |
| 25 |
use File::Path qw( mkpath rmtree ); |
| 26 |
use C4::Biblio qw( GetXmlBiblio ); |
| 27 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
29 |
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML ); |
|
|
30 |
use C4::Biblio qw( GetXmlBiblio ); |
| 28 |
use C4::Items qw( Item2Marc ); |
31 |
use C4::Items qw( Item2Marc ); |
| 29 |
use Koha::RecordProcessor; |
32 |
use Koha::Authorities; |
|
|
33 |
use Koha::Biblios; |
| 30 |
use Koha::Caches; |
34 |
use Koha::Caches; |
| 31 |
use XML::LibXML; |
35 |
use Koha::RecordProcessor; |
|
|
36 |
use Koha::Script; |
| 32 |
|
37 |
|
| 33 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
38 |
use constant LOCK_FILENAME => 'rebuild..LCK'; |
| 34 |
|
39 |
|
|
Lines 65-70
my $run_user = ( getpwuid($<) )[0];
Link Here
|
| 65 |
my $wait_for_lock = 0; |
70 |
my $wait_for_lock = 0; |
| 66 |
my $use_flock; |
71 |
my $use_flock; |
| 67 |
my $table = 'biblioitems'; |
72 |
my $table = 'biblioitems'; |
|
|
73 |
my $forks = 0; |
| 74 |
my $chunk_size = 100000; |
| 68 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
75 |
my $is_memcached = Koha::Caches->get_instance->memcached_cache; |
| 69 |
|
76 |
|
| 70 |
my $verbose_logging = 0; |
77 |
my $verbose_logging = 0; |
|
Lines 93-98
my $result = GetOptions(
Link Here
|
| 93 |
'run-as-root' => \$run_as_root, |
100 |
'run-as-root' => \$run_as_root, |
| 94 |
'wait-for-lock' => \$wait_for_lock, |
101 |
'wait-for-lock' => \$wait_for_lock, |
| 95 |
't|table:s' => \$table, |
102 |
't|table:s' => \$table, |
|
|
103 |
'forks:i' => \$forks, |
| 96 |
); |
104 |
); |
| 97 |
|
105 |
|
| 98 |
if ( not $result or $want_help ) { |
106 |
if ( not $result or $want_help ) { |
|
Lines 409-417
sub index_records {
Link Here
|
| 409 |
mark_zebraqueue_batch_done($entries); |
417 |
mark_zebraqueue_batch_done($entries); |
| 410 |
|
418 |
|
| 411 |
} else { |
419 |
} else { |
| 412 |
my $sth = select_all_records($record_type); |
420 |
$num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize ); |
| 413 |
$num_records_exported = |
|
|
| 414 |
export_marc_records_from_sth( $record_type, $sth, "$directory/$record_type", $nosanitize ); |
| 415 |
unless ($do_not_clear_zebraqueue) { |
421 |
unless ($do_not_clear_zebraqueue) { |
| 416 |
mark_all_zebraqueue_done($record_type); |
422 |
mark_all_zebraqueue_done($record_type); |
| 417 |
} |
423 |
} |
|
Lines 505-511
sub select_all_authorities {
Link Here
|
| 505 |
$strsth .= qq{ WHERE $where } if ($where); |
511 |
$strsth .= qq{ WHERE $where } if ($where); |
| 506 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
512 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
| 507 |
$strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); |
513 |
$strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); |
| 508 |
my $sth = $dbh->prepare($strsth); |
514 |
|
|
|
515 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
| 516 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
| 509 |
$sth->execute(); |
517 |
$sth->execute(); |
| 510 |
return $sth; |
518 |
return $sth; |
| 511 |
} |
519 |
} |
|
Lines 517-532
sub select_all_biblios {
Link Here
|
| 517 |
$strsth .= qq{ WHERE $where } if ($where); |
525 |
$strsth .= qq{ WHERE $where } if ($where); |
| 518 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
526 |
$strsth .= qq{ LIMIT $length } if ( $length && !$offset ); |
| 519 |
$strsth .= qq{ LIMIT $offset,$length } if ($offset); |
527 |
$strsth .= qq{ LIMIT $offset,$length } if ($offset); |
| 520 |
my $sth = $dbh->prepare($strsth); |
528 |
|
|
|
529 |
# If we are forking, we use a new db handle to prevent a potential deadlock |
| 530 |
my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth); |
| 521 |
$sth->execute(); |
531 |
$sth->execute(); |
| 522 |
return $sth; |
532 |
return $sth; |
| 523 |
} |
533 |
} |
| 524 |
|
534 |
|
|
|
535 |
sub export_marc_records { |
| 536 |
my ( $record_type, $directory, $nosanitize ) = @_; |
| 537 |
my $pm = Parallel::ForkManager->new($forks); |
| 538 |
my @original_params = ( $offset, $length ); |
| 539 |
$offset ||= 0; |
| 540 |
$length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count ); |
| 541 |
my $chunk_size = $forks ? ceil( $length / $forks ) : $length; |
| 542 |
my ( $seq, $num_records_exported ) = ( undef, 0 ); |
| 543 |
while ( $chunk_size > 0 ) { |
| 544 |
|
| 545 |
# If you use forks, ->start forks after getting process slot |
| 546 |
unless ( $pm->start ) { |
| 547 |
|
| 548 |
# Child code (or parent when forks parameter is absent) |
| 549 |
$length = $chunk_size; |
| 550 |
my $sth = select_all_records($record_type); |
| 551 |
export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq ); |
| 552 |
$pm->finish; # exit for child only |
| 553 |
} |
| 554 |
$offset += $chunk_size; |
| 555 |
$num_records_exported += $chunk_size; |
| 556 |
$seq++; |
| 557 |
$chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length; |
| 558 |
} |
| 559 |
$pm->wait_all_children; |
| 560 |
( $offset, $length ) = @original_params; # restore for a potential second run (with -a -b) |
| 561 |
return $num_records_exported; |
| 562 |
} |
| 563 |
|
| 525 |
sub export_marc_records_from_sth { |
564 |
sub export_marc_records_from_sth { |
| 526 |
my ( $record_type, $sth, $directory, $nosanitize ) = @_; |
565 |
my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_; |
|
|
566 |
$seq ||= q{}; |
| 527 |
|
567 |
|
| 528 |
my $num_exported = 0; |
568 |
my $num_exported = 0; |
| 529 |
open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!; |
569 |
open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!; |
| 530 |
|
570 |
|
| 531 |
print {$fh} $marcxml_open; |
571 |
print {$fh} $marcxml_open; |
| 532 |
|
572 |
|
| 533 |
- |
|
|