View | Details | Raw Unified | Return to bug 38408
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-17 / +53 lines)
Lines 17-34 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Koha::Script;
20
use Fcntl        qw( LOCK_EX LOCK_NB LOCK_UN );
21
use C4::Context;
21
use File::Temp   qw( tempdir );
22
use File::Path   qw( mkpath rmtree );
22
use Getopt::Long qw( GetOptions );
23
use Getopt::Long qw( GetOptions );
23
use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN );
24
use Parallel::ForkManager;
24
use File::Temp qw( tempdir );
25
use POSIX qw/ceil/;
25
use File::Path qw( mkpath rmtree );
26
use XML::LibXML;
26
use C4::Biblio qw( GetXmlBiblio );
27
28
use C4::Context;
27
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );
29
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );
28
use C4::Items qw( Item2Marc );
30
use C4::Biblio          qw( GetXmlBiblio );
29
use Koha::RecordProcessor;
31
use C4::Items           qw( Item2Marc );
32
use Koha::Authorities;
33
use Koha::Biblios;
30
use Koha::Caches;
34
use Koha::Caches;
31
use XML::LibXML;
35
use Koha::RecordProcessor;
36
use Koha::Script;
32
37
33
use constant LOCK_FILENAME => 'rebuild..LCK';
38
use constant LOCK_FILENAME => 'rebuild..LCK';
34
39
Lines 63-69 my $run_as_root; Link Here
63
my $run_user = (getpwuid($<))[0];
68
my $run_user = (getpwuid($<))[0];
64
my $wait_for_lock = 0;
69
my $wait_for_lock = 0;
65
my $use_flock;
70
my $use_flock;
66
my $table = 'biblioitems';
71
my $table      = 'biblioitems';
72
my $forks      = 0;
73
my $chunk_size = 100000;
67
my $is_memcached = Koha::Caches->get_instance->memcached_cache;
74
my $is_memcached = Koha::Caches->get_instance->memcached_cache;
68
75
69
my $verbose_logging = 0;
76
my $verbose_logging = 0;
Lines 92-97 my $result = GetOptions( Link Here
92
    'run-as-root'   => \$run_as_root,
99
    'run-as-root'   => \$run_as_root,
93
    'wait-for-lock' => \$wait_for_lock,
100
    'wait-for-lock' => \$wait_for_lock,
94
    't|table:s'     => \$table,
101
    't|table:s'     => \$table,
102
    'forks:i'       => \$forks,
95
);
103
);
96
104
97
if (not $result or $want_help) {
105
if (not $result or $want_help) {
Lines 388-395 sub index_records { Link Here
388
            mark_zebraqueue_batch_done($entries);
396
            mark_zebraqueue_batch_done($entries);
389
397
390
        } else {
398
        } else {
391
            my $sth = select_all_records($record_type);
399
            $num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize );
392
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize);
393
            unless ($do_not_clear_zebraqueue) {
400
            unless ($do_not_clear_zebraqueue) {
394
                mark_all_zebraqueue_done($record_type);
401
                mark_all_zebraqueue_done($record_type);
395
            }
402
            }
Lines 474-480 sub select_all_authorities { Link Here
474
    $strsth.=qq{ WHERE $where } if ($where);
481
    $strsth.=qq{ WHERE $where } if ($where);
475
    $strsth.=qq{ LIMIT $length } if ($length && !$offset);
482
    $strsth.=qq{ LIMIT $length } if ($length && !$offset);
476
    $strsth.=qq{ LIMIT $offset,$length } if ($length && $offset);
483
    $strsth.=qq{ LIMIT $offset,$length } if ($length && $offset);
477
    my $sth = $dbh->prepare($strsth);
484
    # If we are forking, we use a new db handle to prevent a potential deadlock
485
    my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);
478
    $sth->execute();
486
    $sth->execute();
479
    return $sth;
487
    return $sth;
480
}
488
}
Lines 486-501 sub select_all_biblios { Link Here
486
    $strsth.=qq{ WHERE $where } if ($where);
494
    $strsth.=qq{ WHERE $where } if ($where);
487
    $strsth.=qq{ LIMIT $length } if ($length && !$offset);
495
    $strsth.=qq{ LIMIT $length } if ($length && !$offset);
488
    $strsth.=qq{ LIMIT $offset,$length } if ($offset);
496
    $strsth.=qq{ LIMIT $offset,$length } if ($offset);
489
    my $sth = $dbh->prepare($strsth);
497
    # If we are forking, we use a new db handle to prevent a potential deadlock
498
    my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);
490
    $sth->execute();
499
    $sth->execute();
491
    return $sth;
500
    return $sth;
492
}
501
}
493
502
503
sub export_marc_records {
504
    my ($record_type, $directory, $nosanitize) = @_;
505
    my $pm = Parallel::ForkManager->new($forks);
506
    my @original_params = ( $offset, $length );
507
    $offset ||= 0;
508
    $length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count );
509
    my $chunk_size = $forks ? ceil( $length / $forks ) : $length;
510
    my ( $seq, $num_records_exported ) = ( undef, 0 );
511
    while ( $chunk_size > 0 ) {
512
        # If you use forks, ->start forks after getting process slot
513
        unless ( $pm->start ) {
514
            # Child code (or parent when forks parameter is absent)
515
            $length = $chunk_size;
516
            my $sth = select_all_records($record_type);
517
            export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq );
518
            $pm->finish;    # exit for child only
519
        }
520
        $offset += $chunk_size;
521
        $num_records_exported += $chunk_size;
522
        $seq++;
523
        $chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length;
524
    }
525
    $pm->wait_all_children;
526
    ( $offset, $length ) = @original_params;    # restore for a potential second run (with -a -b)
527
    return $num_records_exported;
528
}
529
494
sub export_marc_records_from_sth {
530
sub export_marc_records_from_sth {
495
    my ($record_type, $sth, $directory, $nosanitize) = @_;
531
    my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_;
532
    $seq ||= q{};
496
533
497
    my $num_exported = 0;
534
    my $num_exported = 0;
498
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
535
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!;
499
536
500
    print {$fh} $marcxml_open;
537
    print {$fh} $marcxml_open;
501
538
502
- 

Return to bug 38408