| 
      
            Lines 17-34
          
      
      
        Link Here
      
     | 
  
        
          | 17 | 
           | 
          17 | 
           | 
        
        
          | 18 | 
          use Modern::Perl;  | 
          18 | 
          use Modern::Perl;  | 
        
        
          | 19 | 
           | 
          19 | 
           | 
        
          
            
              | 20 | 
              use Koha::Script;  | 
              20 | 
              use Getopt::Long qw( GetOptions );  | 
            
            
               | 
               | 
              21 | 
              use Fcntl        qw( LOCK_EX LOCK_NB LOCK_UN );  | 
            
            
              | 22 | 
              use File::Temp   qw( tempdir );  | 
            
            
              | 23 | 
              use File::Path   qw( mkpath rmtree );  | 
            
            
              | 24 | 
              use Parallel::ForkManager;  | 
            
            
              | 25 | 
              use POSIX qw/ceil/;  | 
            
            
              | 26 | 
              use XML::LibXML;  | 
            
            
              | 27 | 
               | 
            
        
          | 21 | 
          use C4::Context;  | 
          28 | 
          use C4::Context;  | 
        
            
              | 22 | 
              use Getopt::Long        qw( GetOptions );  | 
               | 
               | 
            
            
              | 23 | 
              use Fcntl               qw( LOCK_EX LOCK_NB LOCK_UN );  | 
            
            
              | 24 | 
              use File::Temp          qw( tempdir );  | 
            
            
              | 25 | 
              use File::Path          qw( mkpath rmtree );  | 
            
            
              | 26 | 
              use C4::Biblio          qw( GetXmlBiblio );  | 
            
        
          | 27 | 
          use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );  | 
          29 | 
          use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );  | 
        
            
               | 
               | 
              30 | 
              use C4::Biblio          qw( GetXmlBiblio );  | 
            
        
          | 28 | 
          use C4::Items           qw( Item2Marc );  | 
          31 | 
          use C4::Items           qw( Item2Marc );  | 
        
          
            
              | 29 | 
              use Koha::RecordProcessor;  | 
              32 | 
              use Koha::Authorities;  | 
            
            
               | 
               | 
              33 | 
              use Koha::Biblios;  | 
            
        
          | 30 | 
          use Koha::Caches;  | 
          34 | 
          use Koha::Caches;  | 
        
          
            
              | 31 | 
              use XML::LibXML;  | 
              35 | 
              use Koha::RecordProcessor;  | 
            
            
               | 
               | 
              36 | 
              use Koha::Script;  | 
            
        
          | 32 | 
           | 
          37 | 
           | 
        
        
          | 33 | 
          use constant LOCK_FILENAME => 'rebuild..LCK';  | 
          38 | 
          use constant LOCK_FILENAME => 'rebuild..LCK';  | 
        
        
          | 34 | 
           | 
          39 | 
           | 
        
  
    | 
      
            Lines 65-70
          my $run_user      = ( getpwuid($<) )[0];
      
      
        Link Here
      
     | 
  
        
          | 65 | 
          my $wait_for_lock = 0;  | 
          70 | 
          my $wait_for_lock = 0;  | 
        
        
          | 66 | 
          my $use_flock;  | 
          71 | 
          my $use_flock;  | 
        
        
          | 67 | 
          my $table        = 'biblioitems';  | 
          72 | 
          my $table        = 'biblioitems';  | 
        
            
               | 
               | 
              73 | 
              my $forks        = 0;  | 
            
            
              | 74 | 
              my $chunk_size   = 100000;  | 
            
        
          | 68 | 
          my $is_memcached = Koha::Caches->get_instance->memcached_cache;  | 
          75 | 
          my $is_memcached = Koha::Caches->get_instance->memcached_cache;  | 
        
        
          | 69 | 
           | 
          76 | 
           | 
        
        
          | 70 | 
          my $verbose_logging  = 0;  | 
          77 | 
          my $verbose_logging  = 0;  | 
        
  
    | 
      
            Lines 93-98
          my $result           = GetOptions(
      
      
        Link Here
      
     | 
  
        
          | 93 | 
              'run-as-root'   => \$run_as_root,  | 
          100 | 
              'run-as-root'   => \$run_as_root,  | 
        
        
          | 94 | 
              'wait-for-lock' => \$wait_for_lock,  | 
          101 | 
              'wait-for-lock' => \$wait_for_lock,  | 
        
        
          | 95 | 
              't|table:s'     => \$table,  | 
          102 | 
              't|table:s'     => \$table,  | 
        
            
               | 
               | 
              103 | 
                  'forks:i'       => \$forks,  | 
            
        
          | 96 | 
          );  | 
          104 | 
          );  | 
        
        
          | 97 | 
           | 
          105 | 
           | 
        
        
          | 98 | 
          if ( not $result or $want_help ) { | 
          106 | 
          if ( not $result or $want_help ) { | 
        
  
    | 
      
            Lines 409-417
          sub index_records {
      
      
        Link Here
      
     | 
  
        
          | 409 | 
                      mark_zebraqueue_batch_done($entries);  | 
          417 | 
                      mark_zebraqueue_batch_done($entries);  | 
        
        
          | 410 | 
           | 
          418 | 
           | 
        
        
          | 411 | 
                  } else { | 
          419 | 
                  } else { | 
        
          
            
              | 412 | 
                          my $sth = select_all_records($record_type);  | 
              420 | 
                          $num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize );  | 
            
            
              | 413 | 
                          $num_records_exported =  | 
               | 
               | 
            
            
              | 414 | 
                              export_marc_records_from_sth( $record_type, $sth, "$directory/$record_type", $nosanitize );  | 
            
        
          | 415 | 
                      unless ($do_not_clear_zebraqueue) { | 
          421 | 
                      unless ($do_not_clear_zebraqueue) { | 
        
        
          | 416 | 
                          mark_all_zebraqueue_done($record_type);  | 
          422 | 
                          mark_all_zebraqueue_done($record_type);  | 
        
        
          | 417 | 
                      }  | 
          423 | 
                      }  | 
        
  
    | 
      
            Lines 505-511
          sub select_all_authorities {
      
      
        Link Here
      
     | 
  
        
          | 505 | 
              $strsth .= qq{ WHERE $where }          if ($where); | 
          511 | 
              $strsth .= qq{ WHERE $where }          if ($where); | 
        
        
          | 506 | 
              $strsth .= qq{ LIMIT $length }         if ( $length && !$offset ); | 
          512 | 
              $strsth .= qq{ LIMIT $length }         if ( $length && !$offset ); | 
        
        
          | 507 | 
              $strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); | 
          513 | 
              $strsth .= qq{ LIMIT $offset,$length } if ( $length && $offset ); | 
        
          
            
              | 508 | 
                  my $sth = $dbh->prepare($strsth);  | 
              514 | 
               | 
            
            
               | 
               | 
              515 | 
                  # If we are forking, we use a new db handle to prevent a potential deadlock  | 
            
            
              | 516 | 
                  my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);  | 
            
        
          | 509 | 
              $sth->execute();  | 
          517 | 
              $sth->execute();  | 
        
        
          | 510 | 
              return $sth;  | 
          518 | 
              return $sth;  | 
        
        
          | 511 | 
          }  | 
          519 | 
          }  | 
        
  
    | 
      
            Lines 517-532
          sub select_all_biblios {
      
      
        Link Here
      
     | 
  
        
          | 517 | 
              $strsth .= qq{ WHERE $where }          if ($where); | 
          525 | 
              $strsth .= qq{ WHERE $where }          if ($where); | 
        
        
          | 518 | 
              $strsth .= qq{ LIMIT $length }         if ( $length && !$offset ); | 
          526 | 
              $strsth .= qq{ LIMIT $length }         if ( $length && !$offset ); | 
        
        
          | 519 | 
              $strsth .= qq{ LIMIT $offset,$length } if ($offset); | 
          527 | 
              $strsth .= qq{ LIMIT $offset,$length } if ($offset); | 
        
          
            
              | 520 | 
                  my $sth = $dbh->prepare($strsth);  | 
              528 | 
               | 
            
            
               | 
               | 
              529 | 
                  # If we are forking, we use a new db handle to prevent a potential deadlock  | 
            
            
              | 530 | 
                  my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);  | 
            
        
          | 521 | 
              $sth->execute();  | 
          531 | 
              $sth->execute();  | 
        
        
          | 522 | 
              return $sth;  | 
          532 | 
              return $sth;  | 
        
        
          | 523 | 
          }  | 
          533 | 
          }  | 
        
        
          | 524 | 
           | 
          534 | 
           | 
        
            
               | 
               | 
              535 | 
              sub export_marc_records { | 
            
            
              | 536 | 
                  my ( $record_type, $directory, $nosanitize ) = @_;  | 
            
            
              | 537 | 
                  my $pm              = Parallel::ForkManager->new($forks);  | 
            
            
              | 538 | 
                  my @original_params = ( $offset, $length );  | 
            
            
              | 539 | 
                  $offset ||= 0;  | 
            
            
              | 540 | 
                  $length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count );  | 
            
            
              | 541 | 
                  my $chunk_size = $forks ? ceil( $length / $forks ) : $length;  | 
            
            
              | 542 | 
                  my ( $seq, $num_records_exported ) = ( undef, 0 );  | 
            
            
              | 543 | 
                  while ( $chunk_size > 0 ) { | 
            
            
              | 544 | 
               | 
            
            
              | 545 | 
                      # If you use forks, ->start forks after getting process slot  | 
            
            
              | 546 | 
                      unless ( $pm->start ) { | 
            
            
              | 547 | 
               | 
            
            
              | 548 | 
                          # Child code (or parent when forks parameter is absent)  | 
            
            
              | 549 | 
                          $length = $chunk_size;  | 
            
            
              | 550 | 
                          my $sth = select_all_records($record_type);  | 
            
            
              | 551 | 
                          export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq );  | 
            
            
              | 552 | 
                          $pm->finish;    # exit for child only  | 
            
            
              | 553 | 
                      }  | 
            
            
              | 554 | 
                      $offset               += $chunk_size;  | 
            
            
              | 555 | 
                      $num_records_exported += $chunk_size;  | 
            
            
              | 556 | 
                      $seq++;  | 
            
            
              | 557 | 
                      $chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length;  | 
            
            
              | 558 | 
                  }  | 
            
            
              | 559 | 
                  $pm->wait_all_children;  | 
            
            
              | 560 | 
                  ( $offset, $length ) = @original_params;    # restore for a potential second run (with -a -b)  | 
            
            
              | 561 | 
                  return $num_records_exported;  | 
            
            
              | 562 | 
              }  | 
            
            
              | 563 | 
               | 
            
        
          | 525 | 
          sub export_marc_records_from_sth { | 
          564 | 
          sub export_marc_records_from_sth { | 
        
          
            
              | 526 | 
                  my ( $record_type, $sth, $directory, $nosanitize ) = @_;  | 
              565 | 
                  my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_;  | 
            
            
               | 
               | 
              566 | 
                  $seq ||= q{}; | 
            
        
          | 527 | 
           | 
          567 | 
           | 
        
        
          | 528 | 
              my $num_exported = 0;  | 
          568 | 
              my $num_exported = 0;  | 
        
          
            
              | 529 | 
                  open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;  | 
              569 | 
                  open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!;  | 
            
        
          | 530 | 
           | 
          570 | 
           | 
        
        
          | 531 | 
              print {$fh} $marcxml_open; | 
          571 | 
              print {$fh} $marcxml_open; | 
        
        
          | 532 | 
           | 
          572 | 
           | 
        
            
              | 533 | 
              -   | 
               | 
               |