View | Details | Raw Unified | Return to bug 38408
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-16 / +55 lines)
Lines 17-34 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Koha::Script;
20
use Getopt::Long qw( GetOptions );
21
use Fcntl        qw( LOCK_EX LOCK_NB LOCK_UN );
22
use File::Temp   qw( tempdir );
23
use File::Path   qw( mkpath rmtree );
24
use Parallel::ForkManager;
25
use POSIX qw/ceil/;
26
use XML::LibXML;
27
21
use C4::Context;
28
use C4::Context;
22
use Getopt::Long        qw( GetOptions );
23
use Fcntl               qw( LOCK_EX LOCK_NB LOCK_UN );
24
use File::Temp          qw( tempdir );
25
use File::Path          qw( mkpath rmtree );
26
use C4::Biblio          qw( GetXmlBiblio );
27
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );
29
use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );
30
use C4::Biblio          qw( GetXmlBiblio );
28
use C4::Items           qw( Item2Marc );
31
use C4::Items           qw( Item2Marc );
29
use Koha::RecordProcessor;
32
use Koha::Authorities;
33
use Koha::Biblios;
30
use Koha::Caches;
34
use Koha::Caches;
31
use XML::LibXML;
35
use Koha::RecordProcessor;
36
use Koha::Script;
32
37
33
use constant LOCK_FILENAME => 'rebuild..LCK';
38
use constant LOCK_FILENAME => 'rebuild..LCK';
34
39
Lines 65-70 my $run_user = ( getpwuid($<) )[0]; Link Here
65
my $wait_for_lock = 0;
70
my $wait_for_lock = 0;
66
my $use_flock;
71
my $use_flock;
67
my $table        = 'biblioitems';
72
my $table        = 'biblioitems';
73
my $forks        = 0;
74
my $chunk_size   = 100000;
68
my $is_memcached = Koha::Caches->get_instance->memcached_cache;
75
my $is_memcached = Koha::Caches->get_instance->memcached_cache;
69
76
70
my $verbose_logging  = 0;
77
my $verbose_logging  = 0;
Lines 93-98 my $result = GetOptions( Link Here
93
    'run-as-root'   => \$run_as_root,
100
    'run-as-root'   => \$run_as_root,
94
    'wait-for-lock' => \$wait_for_lock,
101
    'wait-for-lock' => \$wait_for_lock,
95
    't|table:s'     => \$table,
102
    't|table:s'     => \$table,
103
    'forks:i'       => \$forks,
96
);
104
);
97
105
98
if ( not $result or $want_help ) {
106
if ( not $result or $want_help ) {
Lines 409-417 sub index_records { Link Here
409
            mark_zebraqueue_batch_done($entries);
417
            mark_zebraqueue_batch_done($entries);
410
418
411
        } else {
419
        } else {
412
            my $sth = select_all_records($record_type);
420
            $num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize );
413
            $num_records_exported =
414
                export_marc_records_from_sth( $record_type, $sth, "$directory/$record_type", $nosanitize );
415
            unless ($do_not_clear_zebraqueue) {
421
            unless ($do_not_clear_zebraqueue) {
416
                mark_all_zebraqueue_done($record_type);
422
                mark_all_zebraqueue_done($record_type);
417
            }
423
            }
Lines 505-511 sub select_all_authorities { Link Here
505
    $strsth .= qq{ WHERE $where }          if ($where);
511
    $strsth .= qq{ WHERE $where }          if ($where);
506
    $strsth .= qq{ LIMIT $length }         if ( $length && !$offset );
512
    $strsth .= qq{ LIMIT $length }         if ( $length && !$offset );
507
    $strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset );
513
    $strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset );
508
    my $sth = $dbh->prepare($strsth);
514
515
    # If we are forking, we use a new db handle to prevent a potential deadlock
516
    my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);
509
    $sth->execute();
517
    $sth->execute();
510
    return $sth;
518
    return $sth;
511
}
519
}
Lines 517-532 sub select_all_biblios { Link Here
517
    $strsth .= qq{ WHERE $where }          if ($where);
525
    $strsth .= qq{ WHERE $where }          if ($where);
518
    $strsth .= qq{ LIMIT $length }         if ( $length && !$offset );
526
    $strsth .= qq{ LIMIT $length }         if ( $length && !$offset );
519
    $strsth .= qq{ LIMIT $offset,$length } if ($offset);
527
    $strsth .= qq{ LIMIT $offset,$length } if ($offset);
520
    my $sth = $dbh->prepare($strsth);
528
529
    # If we are forking, we use a new db handle to prevent a potential deadlock
530
    my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);
521
    $sth->execute();
531
    $sth->execute();
522
    return $sth;
532
    return $sth;
523
}
533
}
524
534
535
sub export_marc_records {
536
    my ( $record_type, $directory, $nosanitize ) = @_;
537
    my $pm              = Parallel::ForkManager->new($forks);
538
    my @original_params = ( $offset, $length );
539
    $offset ||= 0;
540
    $length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count );
541
    my $chunk_size = $forks ? ceil( $length / $forks ) : $length;
542
    my ( $seq, $num_records_exported ) = ( undef, 0 );
543
    while ( $chunk_size > 0 ) {
544
545
        # If you use forks, ->start forks after getting process slot
546
        unless ( $pm->start ) {
547
548
            # Child code (or parent when forks parameter is absent)
549
            $length = $chunk_size;
550
            my $sth = select_all_records($record_type);
551
            export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq );
552
            $pm->finish;    # exit for child only
553
        }
554
        $offset               += $chunk_size;
555
        $num_records_exported += $chunk_size;
556
        $seq++;
557
        $chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length;
558
    }
559
    $pm->wait_all_children;
560
    ( $offset, $length ) = @original_params;    # restore for a potential second run (with -a -b)
561
    return $num_records_exported;
562
}
563
525
sub export_marc_records_from_sth {
564
sub export_marc_records_from_sth {
526
    my ( $record_type, $sth, $directory, $nosanitize ) = @_;
565
    my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_;
566
    $seq ||= q{};
527
567
528
    my $num_exported = 0;
568
    my $num_exported = 0;
529
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
569
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!;
530
570
531
    print {$fh} $marcxml_open;
571
    print {$fh} $marcxml_open;
532
572
533
- 

Return to bug 38408