| 
      
            Lines 17-34
          
      
      
        Link Here
      
     | 
  
        
          | 17 | 
           | 
          17 | 
           | 
        
        
          | 18 | 
          use Modern::Perl;  | 
          18 | 
          use Modern::Perl;  | 
        
        
          | 19 | 
           | 
          19 | 
           | 
        
          
            
              | 20 | 
              use Koha::Script;  | 
              20 | 
              use Fcntl        qw( LOCK_EX LOCK_NB LOCK_UN );  | 
            
            
              | 21 | 
              use C4::Context;  | 
              21 | 
              use File::Temp   qw( tempdir );  | 
            
            
               | 
               | 
              22 | 
              use File::Path   qw( mkpath rmtree );  | 
            
        
          | 22 | 
          use Getopt::Long qw( GetOptions );  | 
          23 | 
          use Getopt::Long qw( GetOptions );  | 
        
          
            
              | 23 | 
              use Fcntl qw( LOCK_EX LOCK_NB LOCK_UN );  | 
              24 | 
              use Parallel::ForkManager;  | 
            
            
              | 24 | 
              use File::Temp qw( tempdir );  | 
              25 | 
              use POSIX qw/ceil/;  | 
            
            
              | 25 | 
              use File::Path qw( mkpath rmtree );  | 
              26 | 
              use XML::LibXML;  | 
            
            
              | 26 | 
              use C4::Biblio qw( GetXmlBiblio );  | 
              27 | 
               | 
            
            
               | 
               | 
              28 | 
              use C4::Context;  | 
            
        
          | 27 | 
          use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );  | 
          29 | 
          use C4::AuthoritiesMarc qw( GetAuthority GetAuthorityXML );  | 
        
          
            
              | 28 | 
              use C4::Items qw( Item2Marc );  | 
              30 | 
              use C4::Biblio          qw( GetXmlBiblio );  | 
            
            
              | 29 | 
              use Koha::RecordProcessor;  | 
              31 | 
              use C4::Items           qw( Item2Marc );  | 
            
            
               | 
               | 
              32 | 
              use Koha::Authorities;  | 
            
            
              | 33 | 
              use Koha::Biblios;  | 
            
        
          | 30 | 
          use Koha::Caches;  | 
          34 | 
          use Koha::Caches;  | 
        
          
            
              | 31 | 
              use XML::LibXML;  | 
              35 | 
              use Koha::RecordProcessor;  | 
            
            
               | 
               | 
              36 | 
              use Koha::Script;  | 
            
        
          | 32 | 
           | 
          37 | 
           | 
        
        
          | 33 | 
          use constant LOCK_FILENAME => 'rebuild..LCK';  | 
          38 | 
          use constant LOCK_FILENAME => 'rebuild..LCK';  | 
        
        
          | 34 | 
           | 
          39 | 
           | 
        
  
    | 
      
            Lines 63-69
          my $run_as_root;
      
      
        Link Here
      
     | 
  
        
          | 63 | 
          my $run_user = (getpwuid($<))[0];  | 
          68 | 
          my $run_user = (getpwuid($<))[0];  | 
        
        
          | 64 | 
          my $wait_for_lock = 0;  | 
          69 | 
          my $wait_for_lock = 0;  | 
        
        
          | 65 | 
          my $use_flock;  | 
          70 | 
          my $use_flock;  | 
        
          
            
              | 66 | 
              my $table = 'biblioitems';  | 
              71 | 
              my $table      = 'biblioitems';  | 
            
            
               | 
               | 
              72 | 
              my $forks      = 0;  | 
            
            
              | 73 | 
              my $chunk_size = 100000;  | 
            
        
          | 67 | 
          my $is_memcached = Koha::Caches->get_instance->memcached_cache;  | 
          74 | 
          my $is_memcached = Koha::Caches->get_instance->memcached_cache;  | 
        
        
          | 68 | 
           | 
          75 | 
           | 
        
        
          | 69 | 
          my $verbose_logging = 0;  | 
          76 | 
          my $verbose_logging = 0;  | 
        
  
    | 
      
            Lines 92-97
          my $result = GetOptions(
      
      
        Link Here
      
     | 
  
        
          | 92 | 
              'run-as-root'   => \$run_as_root,  | 
          99 | 
              'run-as-root'   => \$run_as_root,  | 
        
        
          | 93 | 
              'wait-for-lock' => \$wait_for_lock,  | 
          100 | 
              'wait-for-lock' => \$wait_for_lock,  | 
        
        
          | 94 | 
              't|table:s'     => \$table,  | 
          101 | 
              't|table:s'     => \$table,  | 
        
            
               | 
               | 
              102 | 
                  'forks:i'       => \$forks,  | 
            
        
          | 95 | 
          );  | 
          103 | 
          );  | 
        
        
          | 96 | 
           | 
          104 | 
           | 
        
        
          | 97 | 
          if (not $result or $want_help) { | 
          105 | 
          if (not $result or $want_help) { | 
        
  
    | 
      
            Lines 388-395
          sub index_records {
      
      
        Link Here
      
     | 
  
        
          | 388 | 
                      mark_zebraqueue_batch_done($entries);  | 
          396 | 
                      mark_zebraqueue_batch_done($entries);  | 
        
        
          | 389 | 
           | 
          397 | 
           | 
        
        
          | 390 | 
                  } else { | 
          398 | 
                  } else { | 
        
          
            
              | 391 | 
                          my $sth = select_all_records($record_type);  | 
              399 | 
                          $num_records_exported = export_marc_records( $record_type, "$directory/$record_type", $nosanitize );  | 
            
            
              | 392 | 
                          $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $nosanitize);  | 
               | 
               | 
            
        
          | 393 | 
                      unless ($do_not_clear_zebraqueue) { | 
          400 | 
                      unless ($do_not_clear_zebraqueue) { | 
        
        
          | 394 | 
                          mark_all_zebraqueue_done($record_type);  | 
          401 | 
                          mark_all_zebraqueue_done($record_type);  | 
        
        
          | 395 | 
                      }  | 
          402 | 
                      }  | 
        
  
    | 
      
            Lines 474-480
          sub select_all_authorities {
      
      
        Link Here
      
     | 
  
        
          | 474 | 
              $strsth.=qq{ WHERE $where } if ($where); | 
          481 | 
              $strsth.=qq{ WHERE $where } if ($where); | 
        
        
          | 475 | 
              $strsth.=qq{ LIMIT $length } if ($length && !$offset); | 
          482 | 
              $strsth.=qq{ LIMIT $length } if ($length && !$offset); | 
        
        
          | 476 | 
              $strsth.=qq{ LIMIT $offset,$length } if ($length && $offset); | 
          483 | 
              $strsth.=qq{ LIMIT $offset,$length } if ($length && $offset); | 
        
          
            
              | 477 | 
                  my $sth = $dbh->prepare($strsth);  | 
              484 | 
                  # If we are forking, we use a new db handle to prevent a potential deadlock  | 
            
            
               | 
               | 
              485 | 
                  my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);  | 
            
        
          | 478 | 
              $sth->execute();  | 
          486 | 
              $sth->execute();  | 
        
        
          | 479 | 
              return $sth;  | 
          487 | 
              return $sth;  | 
        
        
          | 480 | 
          }  | 
          488 | 
          }  | 
        
  
    | 
      
            Lines 486-501
          sub select_all_biblios {
      
      
        Link Here
      
     | 
  
        
          | 486 | 
              $strsth.=qq{ WHERE $where } if ($where); | 
          494 | 
              $strsth.=qq{ WHERE $where } if ($where); | 
        
        
          | 487 | 
              $strsth.=qq{ LIMIT $length } if ($length && !$offset); | 
          495 | 
              $strsth.=qq{ LIMIT $length } if ($length && !$offset); | 
        
        
          | 488 | 
              $strsth.=qq{ LIMIT $offset,$length } if ($offset); | 
          496 | 
              $strsth.=qq{ LIMIT $offset,$length } if ($offset); | 
        
          
            
              | 489 | 
                  my $sth = $dbh->prepare($strsth);  | 
              497 | 
                  # If we are forking, we use a new db handle to prevent a potential deadlock  | 
            
            
               | 
               | 
              498 | 
                  my $sth = $forks ? C4::Context::_new_dbh->prepare($strsth) : $dbh->prepare($strsth);  | 
            
        
          | 490 | 
              $sth->execute();  | 
          499 | 
              $sth->execute();  | 
        
        
          | 491 | 
              return $sth;  | 
          500 | 
              return $sth;  | 
        
        
          | 492 | 
          }  | 
          501 | 
          }  | 
        
        
          | 493 | 
           | 
          502 | 
           | 
        
            
               | 
               | 
              503 | 
              sub export_marc_records { | 
            
            
              | 504 | 
                  my ($record_type, $directory, $nosanitize) = @_;  | 
            
            
              | 505 | 
                  my $pm = Parallel::ForkManager->new($forks);  | 
            
            
              | 506 | 
                  my @original_params = ( $offset, $length );  | 
            
            
              | 507 | 
                  $offset ||= 0;  | 
            
            
              | 508 | 
                  $length ||= ( $record_type eq 'biblio' ? Koha::Biblios->count : Koha::Authorities->count );  | 
            
            
              | 509 | 
                  my $chunk_size = $forks ? ceil( $length / $forks ) : $length;  | 
            
            
              | 510 | 
                  my ( $seq, $num_records_exported ) = ( undef, 0 );  | 
            
            
              | 511 | 
                  while ( $chunk_size > 0 ) { | 
            
            
              | 512 | 
                      # If you use forks, ->start forks after getting process slot  | 
            
            
              | 513 | 
                      unless ( $pm->start ) { | 
            
            
              | 514 | 
                          # Child code (or parent when forks parameter is absent)  | 
            
            
              | 515 | 
                          $length = $chunk_size;  | 
            
            
              | 516 | 
                          my $sth = select_all_records($record_type);  | 
            
            
              | 517 | 
                          export_marc_records_from_sth( $record_type, $sth, "$directory", $nosanitize, $seq );  | 
            
            
              | 518 | 
                          $pm->finish;    # exit for child only  | 
            
            
              | 519 | 
                      }  | 
            
            
              | 520 | 
                      $offset += $chunk_size;  | 
            
            
              | 521 | 
                      $num_records_exported += $chunk_size;  | 
            
            
              | 522 | 
                      $seq++;  | 
            
            
              | 523 | 
                      $chunk_size = $length - $num_records_exported if $num_records_exported + $chunk_size > $length;  | 
            
            
              | 524 | 
                  }  | 
            
            
              | 525 | 
                  $pm->wait_all_children;  | 
            
            
              | 526 | 
                  ( $offset, $length ) = @original_params;    # restore for a potential second run (with -a -b)  | 
            
            
              | 527 | 
                  return $num_records_exported;  | 
            
            
              | 528 | 
              }  | 
            
            
              | 529 | 
               | 
            
        
          | 494 | 
          sub export_marc_records_from_sth { | 
          530 | 
          sub export_marc_records_from_sth { | 
        
          
            
              | 495 | 
                  my ($record_type, $sth, $directory, $nosanitize) = @_;  | 
              531 | 
                  my ( $record_type, $sth, $directory, $nosanitize, $seq ) = @_;  | 
            
            
               | 
               | 
              532 | 
                  $seq ||= q{}; | 
            
        
          | 496 | 
           | 
          533 | 
           | 
        
        
          | 497 | 
              my $num_exported = 0;  | 
          534 | 
              my $num_exported = 0;  | 
        
          
            
              | 498 | 
                  open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;  | 
              535 | 
                  open my $fh, '>:encoding(UTF-8)', "$directory/exported_records$seq" or die $!;  | 
            
        
          | 499 | 
           | 
          536 | 
           | 
        
        
          | 500 | 
              print {$fh} $marcxml_open; | 
          537 | 
              print {$fh} $marcxml_open; | 
        
        
          | 501 | 
           | 
          538 | 
           | 
        
            
              | 502 | 
              -   | 
               | 
               |